From dovecot at dovecot.org Mon Sep 2 17:07:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 02 Sep 2013 17:07:13 +0300 Subject: dovecot-2.2: lib-master: If net_connect_unix() fails with EAGAIN... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a4bfccbc2477 changeset: 16704:a4bfccbc2477 user: Timo Sirainen date: Mon Sep 02 17:06:49 2013 +0300 description: lib-master: If net_connect_unix() fails with EAGAIN, point to a wiki link for reasons. diffstat: src/lib-master/master-auth.c | 4 +++- src/lib-master/master-login.c | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diffs (29 lines): diff -r 470153d92662 -r a4bfccbc2477 src/lib-master/master-auth.c --- a/src/lib-master/master-auth.c Thu Aug 29 16:42:55 2013 +0300 +++ b/src/lib-master/master-auth.c Mon Sep 02 17:06:49 2013 +0300 @@ -186,7 +186,9 @@ conn->fd = net_connect_unix_with_retries(auth->path, SOCKET_CONNECT_RETRY_MSECS); if (conn->fd == -1) { - i_error("net_connect_unix(%s) failed: %m", auth->path); + i_error("net_connect_unix(%s) failed: %m%s", + auth->path, errno != EAGAIN ? "" : + " - http://wiki2.dovecot.org/SocketUnavailable"); master_auth_connection_deinit(&conn); return; } diff -r 470153d92662 -r a4bfccbc2477 src/lib-master/master-login.c --- a/src/lib-master/master-login.c Thu Aug 29 16:42:55 2013 +0300 +++ b/src/lib-master/master-login.c Mon Sep 02 17:06:49 2013 +0300 @@ -307,8 +307,9 @@ fd = net_connect_unix_with_retries(login->postlogin_socket_path, 1000); if (fd == -1) { - i_error("net_connect_unix(%s) failed: %m", - login->postlogin_socket_path); + i_error("net_connect_unix(%s) failed: %m%s", + login->postlogin_socket_path, errno != EAGAIN ? "" : + " - http://wiki2.dovecot.org/SocketUnavailable"); return -1; } From dovecot at dovecot.org Mon Sep 2 17:37:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 02 Sep 2013 17:37:58 +0300 Subject: dovecot-2.2: master: Pre-fork processes only while master doesn'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a3f0aca52d6f changeset: 16705:a3f0aca52d6f user: Timo Sirainen date: Mon Sep 02 17:37:47 2013 +0300 description: master: Pre-fork processes only while master doesn't have more important things to do. diffstat: src/master/service-monitor.c | 54 +++++++++++++++++++++++++++++++++++++++---- src/master/service-process.c | 1 + src/master/service.h | 5 ++++ 3 files changed, 55 insertions(+), 5 deletions(-) diffs (139 lines): diff -r a4bfccbc2477 -r a3f0aca52d6f src/master/service-monitor.c --- a/src/master/service-monitor.c Mon Sep 02 17:06:49 2013 +0300 +++ b/src/master/service-monitor.c Mon Sep 02 17:37:47 2013 +0300 @@ -24,6 +24,7 @@ #define SERVICE_DROP_TIMEOUT_MSECS (10*1000) #define MAX_DIE_WAIT_SECS 5 #define SERVICE_MAX_EXIT_FAILURES_IN_SEC 10 +#define SERVICE_PREFORK_MAX_AT_ONCE 10 static void service_monitor_start_extra_avail(struct service *service); static void service_status_more(struct service_process *process, @@ -81,7 +82,7 @@ service->process_count == service->process_limit) service_login_notify(service, TRUE); - /* we may need to start more */ + /* we may need to start more */ service_monitor_start_extra_avail(service); service_monitor_listen_start(service); } @@ -302,17 +303,18 @@ service_monitor_listen_stop(service); } -static void service_monitor_start_extra_avail(struct service *service) +static bool +service_monitor_start_count(struct service *service, unsigned int limit) { unsigned int i, count; - if (service->process_avail >= service->set->process_min_avail || - service->list->destroying) - return; + i_assert(service->set->process_min_avail >= service->process_avail); count = service->set->process_min_avail - service->process_avail; if (service->process_count + count > service->process_limit) count = service->process_limit - service->process_count; + if (count > limit) + count = limit; for (i = 0; i < count; i++) { if (service_process_create(service) == NULL) { @@ -324,6 +326,44 @@ /* we created some processes, they'll do the listening now */ service_monitor_listen_stop(service); } + return i == count; +} + +static void service_monitor_prefork_timeout(struct service *service) +{ + /* don't prefork more processes if other more important processes had + been forked while we were waiting for this timeout (= master seems + busy) */ + if (service->list->fork_counter != service->prefork_counter) { + service->prefork_counter = service->list->fork_counter; + return; + } + if (service->process_avail < service->set->process_min_avail) { + if (service_monitor_start_count(service, SERVICE_PREFORK_MAX_AT_ONCE) && + service->process_avail < service->set->process_min_avail) + return; + } + timeout_remove(&service->to_prefork); +} + +static void service_monitor_start_extra_avail(struct service *service) +{ + if (service->process_avail >= service->set->process_min_avail || + service->list->destroying) + return; + + if (service->process_avail == 0) { + /* quickly start one process now */ + if (!service_monitor_start_count(service, 1)) + return; + if (service->process_avail >= service->set->process_min_avail) + return; + } + if (service->to_prefork == NULL) { + service->prefork_counter = service->list->fork_counter; + service->to_prefork = + timeout_add_short(0, service_monitor_prefork_timeout, service); + } } static void service_monitor_listen_start_force(struct service *service) @@ -491,6 +531,8 @@ if (service->to_throttle != NULL) timeout_remove(&service->to_throttle); + if (service->to_prefork != NULL) + timeout_remove(&service->to_prefork); } static void services_monitor_wait(struct service_list *service_list) @@ -610,6 +652,8 @@ service_stopped = service->status_fd[0] == -1; if (!service_stopped) { service_monitor_start_extra_avail(service); + /* if there are no longer listening processes, + start listening for more */ if (service->to_throttle == NULL) service_monitor_listen_start(service); } diff -r a4bfccbc2477 -r a3f0aca52d6f src/master/service-process.c --- a/src/master/service-process.c Mon Sep 02 17:06:49 2013 +0300 +++ b/src/master/service-process.c Mon Sep 02 17:37:47 2013 +0300 @@ -295,6 +295,7 @@ } else { pid = fork(); process_forked = TRUE; + service->list->fork_counter++; } if (pid < 0) { diff -r a4bfccbc2477 -r a3f0aca52d6f src/master/service.h --- a/src/master/service.h Mon Sep 02 17:06:49 2013 +0300 +++ b/src/master/service.h Mon Sep 02 17:37:47 2013 +0300 @@ -100,6 +100,10 @@ start dropping pending connections */ struct timeout *to_drop; + /* prefork processes up to process_min_avail if there's time */ + struct timeout *to_prefork; + unsigned int prefork_counter; + /* Last time a "dropping client connections" warning was logged */ time_t last_drop_warning; @@ -120,6 +124,7 @@ pool_t set_pool; int refcount; struct timeout *to_kill; + unsigned int fork_counter; const struct master_settings *set; const struct master_service_settings *service_set; From dovecot at dovecot.org Mon Sep 2 17:45:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 02 Sep 2013 17:45:23 +0300 Subject: dovecot-2.2: master: Fix to previous pre-forking change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/108a8158c1a9 changeset: 16706:108a8158c1a9 user: Timo Sirainen date: Mon Sep 02 17:44:39 2013 +0300 description: master: Fix to previous pre-forking change. diffstat: src/master/service-monitor.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r a3f0aca52d6f -r 108a8158c1a9 src/master/service-monitor.c --- a/src/master/service-monitor.c Mon Sep 02 17:37:47 2013 +0300 +++ b/src/master/service-monitor.c Mon Sep 02 17:44:39 2013 +0300 @@ -360,7 +360,9 @@ return; } if (service->to_prefork == NULL) { - service->prefork_counter = service->list->fork_counter; + /* ioloop handles timeouts before fds (= SIGCHLD callback), + so let the first timeout handler call simply update the fork + counter and the second one check if we're busy or not. */ service->to_prefork = timeout_add_short(0, service_monitor_prefork_timeout, service); } From pigeonhole at rename-it.nl Wed Sep 4 19:12:51 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 04 Sep 2013 18:12:51 +0200 Subject: dovecot-2.2-pigeonhole: doveadm-sieve plugin: Fixed a few crash ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/00fbc2bc1fad changeset: 1794:00fbc2bc1fad user: Stephan Bosch date: Wed Sep 04 18:12:44 2013 +0200 description: doveadm-sieve plugin: Fixed a few crash issues. Patches by Timo Sirainen. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-plugin.c | 28 +++++++++++++++++------ 1 files changed, 20 insertions(+), 8 deletions(-) diffs (93 lines): diff -r 09ce20eaa1c2 -r 00fbc2bc1fad src/plugins/doveadm-sieve/doveadm-sieve-plugin.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Thu Aug 15 21:07:56 2013 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Wed Sep 04 18:12:44 2013 +0200 @@ -105,7 +105,7 @@ MODULE_CONTEXT_SET(user, sieve_user_module, suser); *svstorage_r = suser->sieve_storage; - return 0; + return suser->sieve_storage != NULL ? 1 : 0; } static int sieve_attribute_unset_script(struct mail_storage *storage, @@ -248,7 +248,7 @@ if (value->value != NULL) { type = value->value[0]; - } else { + } else if (value->value_stream != NULL) { ret = i_stream_read_data(value->value_stream, &data, &size, 0); if (ret == -1) { mail_storage_set_critical(storage, "read(%s) failed: %m", @@ -257,6 +257,8 @@ } i_assert(ret > 0); type = data[0]; + } else { + type = MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_SCRIPT; } if (type == MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_LINK) return sieve_attribute_set_active(storage, svstorage, value); @@ -276,10 +278,15 @@ struct sieve_save_context *save_ctx; struct istream *input; const char *scriptname; - int ret = 0; + int ret; - if (mail_sieve_user_init(storage->user, &svstorage) < 0) + if ((ret = mail_sieve_user_init(storage->user, &svstorage)) <= 0) { + if (ret == 0) { + mail_storage_set_error(storage, MAIL_ERROR_NOTFOUND, + "Sieve not enabled for user"); + } return -1; + } if (strcmp(key, MAILBOX_ATTRIBUTE_SIEVE_DEFAULT) == 0) return sieve_attribute_set_default(storage, svstorage, value); @@ -312,6 +319,7 @@ sieve_storage_get_last_error(svstorage, NULL)); return -1; } + ret = 0; while (i_stream_read(input) > 0) { if (sieve_storage_save_continue(save_ctx) < 0) { mail_storage_set_critical(storage, @@ -481,8 +489,8 @@ const char *scriptname, *errstr; int ret; - if (mail_sieve_user_init(storage->user, &svstorage) < 0) - return -1; + if ((ret = mail_sieve_user_init(storage->user, &svstorage)) <= 0) + return ret; if (strcmp(key, MAILBOX_ATTRIBUTE_SIEVE_DEFAULT) == 0) return sieve_attribute_get_default(storage, svstorage, value_r); @@ -547,12 +555,13 @@ { struct mail_user *user = siter->iter.box->storage->user; struct sieve_storage *svstorage; + int ret; if (user->mail_debug) i_debug("doveadm-sieve: Iterating Sieve mailbox attributes"); - if (mail_sieve_user_init(user, &svstorage) < 0) - return -1; + if ((ret = mail_sieve_user_init(user, &svstorage)) <= 0) + return ret; siter->sieve_list = sieve_storage_list_init(svstorage); if (siter->sieve_list == NULL) { @@ -597,6 +606,9 @@ bool active; int ret; + if (siter->sieve_list == NULL) + return NULL; + /* Iterate through all scripts in sieve_dir */ while ((scriptname = sieve_storage_list_next(siter->sieve_list, &active)) != NULL) { From dovecot at dovecot.org Wed Sep 4 21:24:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Sep 2013 21:24:02 +0300 Subject: dovecot-2.2: Fixed off-by-one buffer overflows (practically non-... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/72028f670746 changeset: 16707:72028f670746 user: Timo Sirainen date: Wed Sep 04 21:23:33 2013 +0300 description: Fixed off-by-one buffer overflows (practically non-exploitable). Found by Coverity scan. diffstat: src/doveadm/doveadm-master.c | 2 +- src/master/main.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 108a8158c1a9 -r 72028f670746 src/doveadm/doveadm-master.c --- a/src/doveadm/doveadm-master.c Mon Sep 02 17:44:39 2013 +0300 +++ b/src/doveadm/doveadm-master.c Wed Sep 04 21:23:33 2013 +0300 @@ -23,7 +23,7 @@ i_fatal("open(%s) failed: %m", path); } - ret = read(fd, buf, sizeof(buf)); + ret = read(fd, buf, sizeof(buf)-1); if (ret <= 0) { if (ret == 0) i_error("Empty PID file in %s", path); diff -r 108a8158c1a9 -r 72028f670746 src/master/main.c --- a/src/master/main.c Mon Sep 02 17:44:39 2013 +0300 +++ b/src/master/main.c Wed Sep 04 21:23:33 2013 +0300 @@ -201,7 +201,7 @@ if (fd == -1) return; - ret = read(fd, buf, sizeof(buf)); + ret = read(fd, buf, sizeof(buf)-1); if (ret < 0) i_error("read(%s) failed: %m", path); else { @@ -231,7 +231,7 @@ i_fatal("open(%s) failed: %m", path); } - ret = read(fd, buf, sizeof(buf)); + ret = read(fd, buf, sizeof(buf)-1); if (ret <= 0) { if (ret == 0) i_error("Empty PID file in %s, overriding", path); From dovecot at dovecot.org Wed Sep 4 22:13:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Sep 2013 22:13:18 +0300 Subject: dovecot-2.2: i_close_fd() didn't handle errors correctly with gcc. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/39972f75b143 changeset: 16708:39972f75b143 user: Timo Sirainen date: Wed Sep 04 22:13:01 2013 +0300 description: i_close_fd() didn't handle errors correctly with gcc. diffstat: src/lib/macros.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 72028f670746 -r 39972f75b143 src/lib/macros.h --- a/src/lib/macros.h Wed Sep 04 21:23:33 2013 +0300 +++ b/src/lib/macros.h Wed Sep 04 22:13:01 2013 +0300 @@ -199,7 +199,7 @@ #endif #define i_close_fd(fd) STMT_START { \ - if (unlikely(close_keep_errno(fd)) < 0) \ + if (unlikely(close_keep_errno(fd) < 0)) \ i_error("close(%d[%s:%d]) failed: %m", \ *(fd), __FILE__, __LINE__); \ } STMT_END From dovecot at dovecot.org Wed Sep 4 22:16:09 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Sep 2013 22:16:09 +0300 Subject: dovecot-2.2: lib-index: Fixed modseq updates when modseq was ove... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d297ab881ef changeset: 16709:4d297ab881ef user: Timo Sirainen date: Wed Sep 04 22:16:02 2013 +0300 description: lib-index: Fixed modseq updates when modseq was over 32bits long diffstat: src/lib-index/mail-index-sync-update.c | 2 +- src/lib-index/mail-transaction-log-file.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 39972f75b143 -r 4d297ab881ef src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Wed Sep 04 22:13:01 2013 +0300 +++ b/src/lib-index/mail-index-sync-update.c Wed Sep 04 22:16:02 2013 +0300 @@ -322,7 +322,7 @@ else if (!mail_index_lookup_seq(view, u->uid, &seq)) continue; - min_modseq = ((uint64_t)u->modseq_high32 >> 32) | + min_modseq = ((uint64_t)u->modseq_high32 << 32) | u->modseq_low32; if (highest_modseq < min_modseq) highest_modseq = min_modseq; diff -r 39972f75b143 -r 4d297ab881ef src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Wed Sep 04 22:13:01 2013 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Wed Sep 04 22:16:02 2013 +0300 @@ -1031,7 +1031,7 @@ end = CONST_PTR_OFFSET(data, trans_size - sizeof(*hdr)); for (rec = data; rec < end; rec++) { - uint64_t modseq = ((uint64_t)rec->modseq_high32 >> 32) | + uint64_t modseq = ((uint64_t)rec->modseq_high32 << 32) | rec->modseq_low32; if (*cur_modseq < modseq) *cur_modseq = modseq; From dovecot at dovecot.org Wed Sep 4 22:42:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Sep 2013 22:42:13 +0300 Subject: dovecot-2.2: Fixed file descriptor leaks in rather unimportant p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3091883c672a changeset: 16710:3091883c672a user: Timo Sirainen date: Wed Sep 04 22:41:42 2013 +0300 description: Fixed file descriptor leaks in rather unimportant places. Found by Coverity scan. diffstat: src/doveadm/doveadm-dump-log.c | 1 + src/doveadm/doveadm-dump-mailboxlog.c | 1 + src/doveadm/doveadm-dump-thread.c | 1 + src/lib-master/master-service-settings.c | 2 +- src/lib-storage/index/mbox/mbox-lock.c | 1 + src/lib-storage/mailbox-uidvalidity.c | 4 +++- src/lib/unix-socket-create.c | 1 + src/plugins/acl/acl-backend-vfile-acllist.c | 1 + src/plugins/fts-squat/squat-trie.c | 1 + src/plugins/zlib/zlib-plugin.c | 2 +- src/ssl-params/ssl-params.c | 1 + 11 files changed, 13 insertions(+), 3 deletions(-) diffs (128 lines): diff -r 4d297ab881ef -r 3091883c672a src/doveadm/doveadm-dump-log.c --- a/src/doveadm/doveadm-dump-log.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/doveadm/doveadm-dump-log.c Wed Sep 04 22:41:42 2013 +0300 @@ -524,6 +524,7 @@ ret = dump_record(fd, &modseq); } T_END; } while (ret > 0); + i_close_fd(&fd); } static bool test_dump_log(const char *path) diff -r 4d297ab881ef -r 3091883c672a src/doveadm/doveadm-dump-mailboxlog.c --- a/src/doveadm/doveadm-dump-mailboxlog.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/doveadm/doveadm-dump-mailboxlog.c Wed Sep 04 22:41:42 2013 +0300 @@ -73,6 +73,7 @@ ret = dump_record(fd); } T_END; } while (ret > 0); + i_close_fd(&fd); } static bool test_dump_mailboxlog(const char *path) diff -r 4d297ab881ef -r 3091883c672a src/doveadm/doveadm-dump-thread.c --- a/src/doveadm/doveadm-dump-thread.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/doveadm/doveadm-dump-thread.c Wed Sep 04 22:41:42 2013 +0300 @@ -120,6 +120,7 @@ pos += ret; } T_END; } while (ret > 0); + i_close_fd(&fd); } static bool test_dump_thread(const char *path) diff -r 4d297ab881ef -r 3091883c672a src/lib-master/master-service-settings.c --- a/src/lib-master/master-service-settings.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/lib-master/master-service-settings.c Wed Sep 04 22:41:42 2013 +0300 @@ -388,8 +388,8 @@ if (config_send_request(service, input, fd, path, error_r) == 0) break; + i_close_fd(&fd); if (!retry) { - i_close_fd(&fd); config_exec_fallback(service, input); return -1; } diff -r 4d297ab881ef -r 3091883c672a src/lib-storage/index/mbox/mbox-lock.c --- a/src/lib-storage/index/mbox/mbox-lock.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-lock.c Wed Sep 04 22:41:42 2013 +0300 @@ -290,6 +290,7 @@ if (access(fname, R_OK) < 0) { mail_storage_set_critical(&mbox->storage->storage, "access(%s) failed: %m", box_path); + i_close_fd(&orig_dir_fd); return -1; } } diff -r 4d297ab881ef -r 3091883c672a src/lib-storage/mailbox-uidvalidity.c --- a/src/lib-storage/mailbox-uidvalidity.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/lib-storage/mailbox-uidvalidity.c Wed Sep 04 22:41:42 2013 +0300 @@ -221,8 +221,10 @@ } /* we now have the current uidvalidity value that's hopefully correct */ - if (mailbox_uidvalidity_rename(path, &cur_value, FALSE) < 0) + if (mailbox_uidvalidity_rename(path, &cur_value, FALSE) < 0)?{ + i_close_fd(&fd); return mailbox_uidvalidity_next_rescan(list, path); + } /* fast path succeeded. write the current value to the main uidvalidity file. */ diff -r 4d297ab881ef -r 3091883c672a src/lib/unix-socket-create.c --- a/src/lib/unix-socket-create.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/lib/unix-socket-create.c Wed Sep 04 22:41:42 2013 +0300 @@ -27,6 +27,7 @@ if (chown(path, uid, gid) < 0) { i_error("chown(%s, %s, %s) failed: %m", path, dec2str(uid), dec2str(gid)); + i_close_fd(&fd); return -1; } } diff -r 4d297ab881ef -r 3091883c672a src/plugins/acl/acl-backend-vfile-acllist.c --- a/src/plugins/acl/acl-backend-vfile-acllist.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/plugins/acl/acl-backend-vfile-acllist.c Wed Sep 04 22:41:42 2013 +0300 @@ -139,6 +139,7 @@ i_error("Broken acllist file: %s", path); if (unlink(path) < 0 && errno != ENOENT) i_error("unlink(%s) failed: %m", path); + i_close_fd(&fd); return -1; } acllist.name = p_strdup(backend->acllist_pool, p + 1); diff -r 4d297ab881ef -r 3091883c672a src/plugins/fts-squat/squat-trie.c --- a/src/plugins/fts-squat/squat-trie.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/plugins/fts-squat/squat-trie.c Wed Sep 04 22:41:42 2013 +0300 @@ -1556,6 +1556,7 @@ if (fchown(fd, (uid_t)-1, trie->create_gid) < 0) { i_error("fchown(%s, -1, %ld) failed: %m", path, (long)trie->create_gid); + i_close_fd(&fd); return -1; } } diff -r 4d297ab881ef -r 3091883c672a src/plugins/zlib/zlib-plugin.c --- a/src/plugins/zlib/zlib-plugin.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/plugins/zlib/zlib-plugin.c Wed Sep 04 22:41:42 2013 +0300 @@ -312,7 +312,7 @@ i_close_fd(&fd); return 0; } - input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); + input = i_stream_create_fd(fd, MAX_INBUF_SIZE, TRUE); i_stream_set_name(input, box_path); box->input = handler->create_istream(input, TRUE); i_stream_unref(&input); diff -r 4d297ab881ef -r 3091883c672a src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Wed Sep 04 22:16:02 2013 +0300 +++ b/src/ssl-params/ssl-params.c Wed Sep 04 22:41:42 2013 +0300 @@ -170,6 +170,7 @@ } if (st.st_size == 0 || st.st_size > MAX_PARAM_FILE_SIZE) { i_error("Corrupted file: %s", param->path); + i_close_fd(&fd); (void)unlink(param->path); return -1; } From dovecot at dovecot.org Wed Sep 4 22:44:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 04 Sep 2013 22:44:02 +0300 Subject: dovecot-2.2: And removed accidentally committed nbsp. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ffe37152ed0 changeset: 16711:6ffe37152ed0 user: Timo Sirainen date: Wed Sep 04 22:43:47 2013 +0300 description: And removed accidentally committed nbsp. diffstat: src/lib-storage/mailbox-uidvalidity.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3091883c672a -r 6ffe37152ed0 src/lib-storage/mailbox-uidvalidity.c --- a/src/lib-storage/mailbox-uidvalidity.c Wed Sep 04 22:41:42 2013 +0300 +++ b/src/lib-storage/mailbox-uidvalidity.c Wed Sep 04 22:43:47 2013 +0300 @@ -221,7 +221,7 @@ } /* we now have the current uidvalidity value that's hopefully correct */ - if (mailbox_uidvalidity_rename(path, &cur_value, FALSE) < 0)?{ + if (mailbox_uidvalidity_rename(path, &cur_value, FALSE) < 0) { i_close_fd(&fd); return mailbox_uidvalidity_next_rescan(list, path); } From dovecot at dovecot.org Thu Sep 5 21:07:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 05 Sep 2013 21:07:02 +0300 Subject: dovecot-2.2: lib-lda: Fixed passing through error message when m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/82975fb3ae44 changeset: 16712:82975fb3ae44 user: Timo Sirainen date: Thu Sep 05 21:06:50 2013 +0300 description: lib-lda: Fixed passing through error message when mailbox couldn't be opened. diffstat: src/lib-lda/mail-deliver.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 6ffe37152ed0 -r 82975fb3ae44 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Wed Sep 04 22:43:47 2013 +0300 +++ b/src/lib-lda/mail-deliver.c Thu Sep 05 21:06:50 2013 +0300 @@ -282,8 +282,10 @@ mailbox_name = str_sanitize(mailbox, 80); if (mail_deliver_save_open(&open_ctx, mailbox, &box, &error, &errstr) < 0) { - if (box != NULL) + if (box != NULL) { + *storage_r = mailbox_get_storage(box); mailbox_free(&box); + } mail_deliver_log(ctx, "save failed to open mailbox %s: %s", mailbox_name, errstr); return -1; From dovecot at dovecot.org Fri Sep 6 02:40:10 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 06 Sep 2013 02:40:10 +0300 Subject: dovecot-2.2: lib-index: Minor fix to regenerating missing transa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00d0856b4a0b changeset: 16713:00d0856b4a0b user: Timo Sirainen date: Fri Sep 06 02:39:16 2013 +0300 description: lib-index: Minor fix to regenerating missing transaction log parts. diffstat: src/lib-index/mail-index-view-sync.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 82975fb3ae44 -r 00d0856b4a0b src/lib-index/mail-index-view-sync.c --- a/src/lib-index/mail-index-view-sync.c Thu Sep 05 21:06:50 2013 +0300 +++ b/src/lib-index/mail-index-view-sync.c Fri Sep 06 02:39:16 2013 +0300 @@ -273,7 +273,7 @@ qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp); qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp); - return memcmp(old_idx, new_idx, old_count * sizeof(old_idx)) == 0; + return memcmp(old_idx, new_idx, old_count * sizeof(*old_idx)) == 0; } static int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx, From dovecot at dovecot.org Fri Sep 6 02:40:10 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 06 Sep 2013 02:40:10 +0300 Subject: dovecot-2.2: Avoid doing side effects in assert. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/209a1ef01426 changeset: 16714:209a1ef01426 user: Timo Sirainen date: Fri Sep 06 02:39:56 2013 +0300 description: Avoid doing side effects in assert. diffstat: src/lib-imap-client/imapc-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 00d0856b4a0b -r 209a1ef01426 src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Fri Sep 06 02:39:16 2013 +0300 +++ b/src/lib-imap-client/imapc-connection.c Fri Sep 06 02:39:56 2013 +0300 @@ -1854,7 +1854,8 @@ /* %1s - no quoting */ const char *arg = va_arg(args, const char *); - i_assert(cmd_fmt[++i] == 's'); + i++; + i_assert(cmd_fmt[i] == 's'); str_append(cmd->data, arg); break; } From dovecot at dovecot.org Fri Sep 6 18:29:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 06 Sep 2013 18:29:11 +0300 Subject: dovecot-2.2: fts-lucene: Fixed crash in doveadm dumping lucene i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07a8718ef3a8 changeset: 16715:07a8718ef3a8 user: Timo Sirainen date: Fri Sep 06 18:26:48 2013 +0300 description: fts-lucene: Fixed crash in doveadm dumping lucene index. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (19 lines): diff -r 209a1ef01426 -r 07a8718ef3a8 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Fri Sep 06 02:39:56 2013 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Fri Sep 06 18:26:48 2013 +0300 @@ -110,11 +110,11 @@ index = i_new(struct lucene_index, 1); index->path = i_strdup(path); index->list = list; - index->normalizer = !set->normalize ? NULL : - mailbox_list_get_namespace(list)->user->default_normalizer; - if (set != NULL) + if (set != NULL) { index->set = *set; - else { + index->normalizer = !set->normalize ? NULL : + mailbox_list_get_namespace(list)->user->default_normalizer; + } else { /* this is valid only for doveadm dump, so it doesn't matter */ index->set.default_language = ""; } From dovecot at dovecot.org Fri Sep 6 18:29:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 06 Sep 2013 18:29:11 +0300 Subject: dovecot-2.2: imapc: Don't crash if server sends EXISTS while mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b41828d3f663 changeset: 16716:b41828d3f663 user: Timo Sirainen date: Fri Sep 06 18:28:59 2013 +0300 description: imapc: Don't crash if server sends EXISTS while mailbox isn't selected. diffstat: src/lib-storage/index/imapc/imapc-mailbox.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 07a8718ef3a8 -r b41828d3f663 src/lib-storage/index/imapc/imapc-mailbox.c --- a/src/lib-storage/index/imapc/imapc-mailbox.c Fri Sep 06 18:26:48 2013 +0300 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Fri Sep 06 18:28:59 2013 +0300 @@ -127,13 +127,14 @@ imapc_untagged_exists(const struct imapc_untagged_reply *reply, struct imapc_mailbox *mbox) { - struct mail_index_view *view = mbox->delayed_sync_view; + struct mail_index_view *view; uint32_t exists_count = reply->num; const struct mail_index_header *hdr; if (mbox == NULL) return; + view = mbox->delayed_sync_view; if (view == NULL) view = imapc_mailbox_get_sync_view(mbox); From dovecot at dovecot.org Fri Sep 6 18:31:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 06 Sep 2013 18:31:04 +0300 Subject: dovecot-2.2: auth: Removed unnecessary NULL check. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/66b5839375f0 changeset: 16717:66b5839375f0 user: Timo Sirainen date: Fri Sep 06 18:30:54 2013 +0300 description: auth: Removed unnecessary NULL check. diffstat: src/auth/auth-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b41828d3f663 -r 66b5839375f0 src/auth/auth-request.c --- a/src/auth/auth-request.c Fri Sep 06 18:28:59 2013 +0300 +++ b/src/auth/auth-request.c Fri Sep 06 18:30:54 2013 +0300 @@ -60,7 +60,7 @@ request->set = global_auth_settings; request->mech = mech; - request->mech_name = mech == NULL ? NULL : mech->mech_name; + request->mech_name = mech->mech_name; request->extra_fields = auth_fields_init(request->pool); return request; } From dovecot at dovecot.org Sun Sep 8 19:20:50 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 08 Sep 2013 19:20:50 +0300 Subject: dovecot-2.2: hmac: Fixed crashes on CPUs that don't allow unalig... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/92a54bb2ee00 changeset: 16718:92a54bb2ee00 user: Timo Sirainen date: Sun Sep 08 19:20:39 2013 +0300 description: hmac: Fixed crashes on CPUs that don't allow unaligned memory access. diffstat: src/lib/hmac-cram-md5.c | 6 ++++-- src/lib/hmac.c | 7 +++++-- src/lib/hmac.h | 13 +++++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diffs (89 lines): diff -r 66b5839375f0 -r 92a54bb2ee00 src/lib/hmac-cram-md5.c --- a/src/lib/hmac-cram-md5.c Fri Sep 06 18:30:54 2013 +0300 +++ b/src/lib/hmac-cram-md5.c Sun Sep 08 19:20:39 2013 +0300 @@ -9,9 +9,10 @@ #include "md5.h" #include "hmac-cram-md5.h" -void hmac_md5_get_cram_context(struct hmac_context *hmac_ctx, +void hmac_md5_get_cram_context(struct hmac_context *_hmac_ctx, unsigned char context_digest[CRAM_MD5_CONTEXTLEN]) { + struct hmac_context_priv *hmac_ctx = &_hmac_ctx->u.priv; unsigned char *cdp; struct md5_context *ctx = (void*)hmac_ctx->ctx; @@ -34,9 +35,10 @@ CDPUT(cdp, ctx->d); } -void hmac_md5_set_cram_context(struct hmac_context *hmac_ctx, +void hmac_md5_set_cram_context(struct hmac_context *_hmac_ctx, const unsigned char context_digest[CRAM_MD5_CONTEXTLEN]) { + struct hmac_context_priv *hmac_ctx = &_hmac_ctx->u.priv; const unsigned char *cdp; struct md5_context *ctx = (void*)hmac_ctx->ctx; diff -r 66b5839375f0 -r 92a54bb2ee00 src/lib/hmac.c --- a/src/lib/hmac.c Fri Sep 06 18:30:54 2013 +0300 +++ b/src/lib/hmac.c Sun Sep 08 19:20:39 2013 +0300 @@ -11,9 +11,10 @@ #include "hmac.h" #include "safe-memset.h" -void hmac_init(struct hmac_context *ctx, const unsigned char *key, +void hmac_init(struct hmac_context *_ctx, const unsigned char *key, size_t key_len, const struct hash_method *meth) { + struct hmac_context_priv *ctx = &_ctx->u.priv; int i; unsigned char k_ipad[64]; unsigned char k_opad[64]; @@ -49,8 +50,10 @@ safe_memset(k_opad, 0, 64); } -void hmac_final(struct hmac_context *ctx, unsigned char *digest) +void hmac_final(struct hmac_context *_ctx, unsigned char *digest) { + struct hmac_context_priv *ctx = &_ctx->u.priv; + ctx->hash->result(ctx->ctx, digest); ctx->hash->loop(ctx->ctxo, digest, ctx->hash->digest_size); diff -r 66b5839375f0 -r 92a54bb2ee00 src/lib/hmac.h --- a/src/lib/hmac.h Fri Sep 06 18:30:54 2013 +0300 +++ b/src/lib/hmac.h Sun Sep 08 19:20:39 2013 +0300 @@ -6,20 +6,29 @@ #define HMAC_MAX_CONTEXT_SIZE 256 -struct hmac_context { +struct hmac_context_priv { char ctx[HMAC_MAX_CONTEXT_SIZE]; char ctxo[HMAC_MAX_CONTEXT_SIZE]; const struct hash_method *hash; }; +struct hmac_context { + union { + struct hmac_context_priv priv; + uint64_t padding_requirement; + } u; +}; + void hmac_init(struct hmac_context *ctx, const unsigned char *key, size_t key_len, const struct hash_method *meth); void hmac_final(struct hmac_context *ctx, unsigned char *digest); static inline void -hmac_update(struct hmac_context *ctx, const void *data, size_t size) +hmac_update(struct hmac_context *_ctx, const void *data, size_t size) { + struct hmac_context_priv *ctx = &_ctx->u.priv; + ctx->hash->loop(ctx->ctx, data, size); } From dovecot at dovecot.org Mon Sep 9 04:35:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Sep 2013 04:35:23 +0300 Subject: dovecot-2.2: Minor code cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63edc752479d changeset: 16719:63edc752479d user: Timo Sirainen date: Mon Sep 09 04:34:45 2013 +0300 description: Minor code cleanup. diffstat: src/lib/ostream-hash.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 92a54bb2ee00 -r 63edc752479d src/lib/ostream-hash.h --- a/src/lib/ostream-hash.h Sun Sep 08 19:20:39 2013 +0300 +++ b/src/lib/ostream-hash.h Mon Sep 09 04:34:45 2013 +0300 @@ -1,6 +1,8 @@ #ifndef OSTREAM_HASH_H #define OSTREAM_HASH_H +struct hash_method; + /* hash_context must be allocated and initialized by caller. This ostream will simply call method->loop() for all the data going through the ostream. */ struct ostream * From dovecot at dovecot.org Mon Sep 9 04:35:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 09 Sep 2013 04:35:23 +0300 Subject: dovecot-2.2: liblib: Added istream-hash, similar to ostream-hash. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f73c388ad360 changeset: 16720:f73c388ad360 user: Timo Sirainen date: Mon Sep 09 04:34:58 2013 +0300 description: liblib: Added istream-hash, similar to ostream-hash. diffstat: src/lib/Makefile.am | 2 + src/lib/istream-hash.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/istream-hash.h | 12 ++++++++++ 3 files changed, 74 insertions(+), 0 deletions(-) diffs (99 lines): diff -r 63edc752479d -r f73c388ad360 src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Sep 09 04:34:45 2013 +0300 +++ b/src/lib/Makefile.am Mon Sep 09 04:34:58 2013 +0300 @@ -62,6 +62,7 @@ istream-crlf.c \ istream-data.c \ istream-file.c \ + istream-hash.c \ istream-jsonstr.c \ istream-limit.c \ istream-mmap.c \ @@ -189,6 +190,7 @@ istream-chain.h \ istream-concat.h \ istream-crlf.h \ + istream-hash.h \ istream-jsonstr.h \ istream-private.h \ istream-rawlog.h \ diff -r 63edc752479d -r f73c388ad360 src/lib/istream-hash.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-hash.c Mon Sep 09 04:34:58 2013 +0300 @@ -0,0 +1,60 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash-method.h" +#include "istream-private.h" +#include "istream-hash.h" + +struct hash_istream { + struct istream_private istream; + + const struct hash_method *method; + void *hash_context; +}; + +static ssize_t +i_stream_hash_read(struct istream_private *stream) +{ + struct hash_istream *hstream = (struct hash_istream *)stream; + const unsigned char *data; + size_t size; + ssize_t ret; + + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + + ret = i_stream_read_copy_from_parent(&stream->istream); + if (ret > 0 && hstream->hash_context != NULL) { + data = i_stream_get_data(&stream->istream, &size); + i_assert((size_t)ret <= size); + hstream->method->loop(hstream->hash_context, + data+(size-ret), ret); + } else if (ret < 0) { + /* we finished hashing it. don't access it anymore, because + the memory pointed by the hash may be freed before the + istream itself */ + hstream->hash_context = NULL; + } + return ret; +} + +struct istream * +i_stream_create_hash(struct istream *input, const struct hash_method *method, + void *hash_context) +{ + struct hash_istream *hstream; + + hstream = i_new(struct hash_istream, 1); + hstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + hstream->istream.stream_size_passthrough = TRUE; + + hstream->istream.read = i_stream_hash_read; + + hstream->istream.istream.blocking = input->blocking; + hstream->istream.istream.seekable = FALSE; + + hstream->method = method; + hstream->hash_context = hash_context; + return i_stream_create(&hstream->istream, input, + i_stream_get_fd(input)); +} diff -r 63edc752479d -r f73c388ad360 src/lib/istream-hash.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-hash.h Mon Sep 09 04:34:58 2013 +0300 @@ -0,0 +1,12 @@ +#ifndef ISTREAM_HASH_H +#define ISTREAM_HASH_H + +struct hash_method; + +/* hash_context must be allocated and initialized by caller. This istream will + simply call method->loop() for all the data going through the istream. */ +struct istream * +i_stream_create_hash(struct istream *input, const struct hash_method *method, + void *hash_context); + +#endif From dovecot at dovecot.org Tue Sep 10 04:11:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 10 Sep 2013 04:11:30 +0300 Subject: dovecot-2.2: lib-fs: Added support for giving a hash of the writ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7e0ebc2faf3c changeset: 16721:7e0ebc2faf3c user: Timo Sirainen date: Tue Sep 10 00:27:12 2013 +0300 description: lib-fs: Added support for giving a hash of the written data to be verified by storage. diffstat: src/lib-fs/fs-api-private.h | 3 +++ src/lib-fs/fs-api.c | 17 +++++++++++++++++ src/lib-fs/fs-api.h | 13 ++++++++++++- 3 files changed, 32 insertions(+), 1 deletions(-) diffs (105 lines): diff -r f73c388ad360 -r 7e0ebc2faf3c src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Sep 09 04:34:58 2013 +0300 +++ b/src/lib-fs/fs-api-private.h Tue Sep 10 00:27:12 2013 +0300 @@ -75,6 +75,9 @@ struct istream *pending_read_input; bool write_pending; + const struct hash_method *write_digest_method; + void *write_digest; + pool_t metadata_pool; ARRAY_TYPE(fs_metadata) metadata; diff -r f73c388ad360 -r 7e0ebc2faf3c src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Sep 09 04:34:58 2013 +0300 +++ b/src/lib-fs/fs-api.c Tue Sep 10 00:27:12 2013 +0300 @@ -4,6 +4,7 @@ #include "array.h" #include "module-dir.h" #include "str.h" +#include "hash-method.h" #include "istream.h" #include "istream-seekable.h" #include "ostream.h" @@ -199,6 +200,7 @@ i_stream_unref(&file->copy_input); (void)fs_write_stream_abort(file, &file->copy_output); } + i_free_and_null(file->write_digest); if (file->fs->v.file_close != NULL) T_BEGIN { file->fs->v.file_close(file); } T_END; @@ -251,6 +253,11 @@ file->fs->v.get_path(file); } +struct fs *fs_file_fs(struct fs_file *file) +{ + return file->fs; +} + static void ATTR_FORMAT(2, 0) fs_set_verror(struct fs *fs, const char *fmt, va_list args) { @@ -475,6 +482,16 @@ } T_END; } +void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, + const void *digest) +{ + file->write_digest_method = method; + + i_free(file->write_digest); + file->write_digest = i_malloc(method->digest_size); + memcpy(file->write_digest, digest, method->digest_size); +} + void fs_file_set_async_callback(struct fs_file *file, fs_file_async_callback_t *callback, void *context) diff -r f73c388ad360 -r 7e0ebc2faf3c src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Mon Sep 09 04:34:58 2013 +0300 +++ b/src/lib-fs/fs-api.h Tue Sep 10 00:27:12 2013 +0300 @@ -5,6 +5,7 @@ struct fs; struct fs_file; struct fs_lock; +struct hash_method; enum fs_properties { FS_PROPERTY_METADATA = 0x01, @@ -19,7 +20,9 @@ FS_PROPERTY_RELIABLEITER= 0x40, /* Backend uses directories, which aren't automatically deleted when its children are deleted. */ - FS_PROPERTY_DIRECTORIES = 0x80 + FS_PROPERTY_DIRECTORIES = 0x80, + FS_PROPERTY_WRITE_HASH_MD5 = 0x100, + FS_PROPERTY_WRITE_HASH_SHA256 = 0x200 }; enum fs_open_mode { @@ -115,6 +118,8 @@ FS_OPEN_MODE_CREATE_UNIQUE_128 and the write has already finished, return the path including the generated filename. */ const char *fs_file_path(struct fs_file *file); +/* Returns the file's fs. */ +struct fs *fs_file_fs(struct fs_file *file); /* Return the error message for the last failed operation. */ const char *fs_last_error(struct fs *fs); @@ -151,6 +156,12 @@ /* Abort writing via stream. Anything written to the stream is discarded. */ void fs_write_stream_abort(struct fs_file *file, struct ostream **output); +/* Set a hash to the following write. The storage can then verify that the + input data matches the specified hash, or fail if it doesn't. Typically + implemented by Content-MD5 header. */ +void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, + const void *digest); + /* Call the specified callback whenever the file can be read/written to. May call the callback immediately. */ void fs_file_set_async_callback(struct fs_file *file, From dovecot at dovecot.org Tue Sep 10 04:11:30 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 10 Sep 2013 04:11:30 +0300 Subject: dovecot-2.2: lib-fs: Added support for asynchronous fs iteration. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ed90c968ab1 changeset: 16722:6ed90c968ab1 user: Timo Sirainen date: Tue Sep 10 04:11:03 2013 +0300 description: lib-fs: Added support for asynchronous fs iteration. diffstat: src/lib-fs/fs-api-private.h | 4 ++++ src/lib-fs/fs-api.c | 13 +++++++++++++ src/lib-fs/fs-api.h | 13 ++++++++++++- 3 files changed, 29 insertions(+), 1 deletions(-) diffs (65 lines): diff -r 7e0ebc2faf3c -r 6ed90c968ab1 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Tue Sep 10 00:27:12 2013 +0300 +++ b/src/lib-fs/fs-api-private.h Tue Sep 10 04:11:03 2013 +0300 @@ -93,6 +93,10 @@ struct fs_iter { struct fs *fs; enum fs_iter_flags flags; + + bool async_have_more; + fs_file_async_callback_t *async_callback; + void *async_context; }; extern const struct fs fs_class_posix; diff -r 7e0ebc2faf3c -r 6ed90c968ab1 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Tue Sep 10 00:27:12 2013 +0300 +++ b/src/lib-fs/fs-api.c Tue Sep 10 04:11:03 2013 +0300 @@ -670,6 +670,19 @@ return iter->fs->v.iter_next(iter); } +void fs_iter_set_async_callback(struct fs_iter *iter, + fs_file_async_callback_t *callback, + void *context) +{ + iter->async_callback = callback; + iter->async_context = context; +} + +bool fs_iter_have_more(struct fs_iter *iter) +{ + return iter->async_have_more; +} + void fs_set_error(struct fs *fs, const char *fmt, ...) { va_list args; diff -r 7e0ebc2faf3c -r 6ed90c968ab1 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Tue Sep 10 00:27:12 2013 +0300 +++ b/src/lib-fs/fs-api.h Tue Sep 10 04:11:03 2013 +0300 @@ -59,7 +59,9 @@ enum fs_iter_flags { /* Iterate only directories, not files */ - FS_ITER_FLAG_DIRS = 0x01 + FS_ITER_FLAG_DIRS = 0x01, + /* Request asynchronous iteration. */ + FS_ITER_FLAG_ASYNC = 0x02 }; struct fs_settings { @@ -206,4 +208,13 @@ /* Returns the next filename. */ const char *fs_iter_next(struct fs_iter *iter); +/* For asynchronous iterations: Specify the callback that is called whenever + there's more data available for reading. */ +void fs_iter_set_async_callback(struct fs_iter *iter, + fs_file_async_callback_t *callback, + void *context); +/* For asynchronous iterations: If fs_iter_next() returns NULL, use this + function to determine if you should wait for more data or finish up. */ +bool fs_iter_have_more(struct fs_iter *iter); + #endif From dovecot at dovecot.org Thu Sep 12 01:39:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Sep 2013 01:39:07 +0300 Subject: dovecot-2.2: auth: auth-client socket should be owned by $defaul... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/96fa57cacfc7 changeset: 16723:96fa57cacfc7 user: Timo Sirainen date: Thu Sep 12 01:38:54 2013 +0300 description: auth: auth-client socket should be owned by $default_internal_user It's mostly the same as auth-login, which is also owned by it. diffstat: src/auth/auth-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6ed90c968ab1 -r 96fa57cacfc7 src/auth/auth-settings.c --- a/src/auth/auth-settings.c Tue Sep 10 04:11:03 2013 +0300 +++ b/src/auth/auth-settings.c Thu Sep 12 01:38:54 2013 +0300 @@ -19,7 +19,7 @@ { "login/login", 0666, "", "" }, { "token-login/tokenlogin", 0666, "", "" }, { "auth-login", 0600, "$default_internal_user", "" }, - { "auth-client", 0600, "", "" }, + { "auth-client", 0600, "$default_internal_user", "" }, { "auth-userdb", 0666, "$default_internal_user", "" }, { "auth-master", 0600, "", "" } }; From dovecot at dovecot.org Thu Sep 12 03:00:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Sep 2013 03:00:26 +0300 Subject: dovecot-2.2: istream-hash: Allow seeking, but only after reading... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c8f17405ff71 changeset: 16724:c8f17405ff71 user: Timo Sirainen date: Thu Sep 12 02:32:47 2013 +0300 description: istream-hash: Allow seeking, but only after reading the hash. diffstat: src/lib/istream-hash.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 96fa57cacfc7 -r c8f17405ff71 src/lib/istream-hash.c --- a/src/lib/istream-hash.c Thu Sep 12 01:38:54 2013 +0300 +++ b/src/lib/istream-hash.c Thu Sep 12 02:32:47 2013 +0300 @@ -38,6 +38,20 @@ return ret; } +static void +i_stream_hash_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + struct hash_istream *hstream = (struct hash_istream *)stream; + + if (hstream->hash_context != NULL) { + /* we support seeking only after the hash is finished */ + stream->istream.stream_errno = ESPIPE; + } + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; +} + struct istream * i_stream_create_hash(struct istream *input, const struct hash_method *method, void *hash_context) @@ -49,9 +63,10 @@ hstream->istream.stream_size_passthrough = TRUE; hstream->istream.read = i_stream_hash_read; + hstream->istream.seek = i_stream_hash_seek; hstream->istream.istream.blocking = input->blocking; - hstream->istream.istream.seekable = FALSE; + hstream->istream.istream.seekable = input->seekable; hstream->method = method; hstream->hash_context = hash_context; From dovecot at dovecot.org Thu Sep 12 03:00:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Sep 2013 03:00:26 +0300 Subject: dovecot-2.2: istream: Improved "stream not seekable" panic message. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/262fbb878ed4 changeset: 16725:262fbb878ed4 user: Timo Sirainen date: Thu Sep 12 02:33:01 2013 +0300 description: istream: Improved "stream not seekable" panic message. diffstat: src/lib/istream.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r c8f17405ff71 -r 262fbb878ed4 src/lib/istream.c --- a/src/lib/istream.c Thu Sep 12 02:32:47 2013 +0300 +++ b/src/lib/istream.c Thu Sep 12 02:33:01 2013 +0300 @@ -634,7 +634,8 @@ size_t available; if (stream->istream.v_offset > v_offset) - i_panic("stream doesn't support seeking backwards"); + i_panic("stream %s doesn't support seeking backwards", + i_stream_get_name(&stream->istream)); while (stream->istream.v_offset < v_offset) { (void)i_stream_read(&stream->istream); From dovecot at dovecot.org Thu Sep 12 03:00:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Sep 2013 03:00:26 +0300 Subject: dovecot-2.2: istream-hash: Data was hashed multiple times in som... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fe8ac44541ae changeset: 16726:fe8ac44541ae user: Timo Sirainen date: Thu Sep 12 02:58:51 2013 +0300 description: istream-hash: Data was hashed multiple times in some situations diffstat: src/lib/istream-hash.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 262fbb878ed4 -r fe8ac44541ae src/lib/istream-hash.c --- a/src/lib/istream-hash.c Thu Sep 12 02:33:01 2013 +0300 +++ b/src/lib/istream-hash.c Thu Sep 12 02:58:51 2013 +0300 @@ -10,6 +10,7 @@ const struct hash_method *method; void *hash_context; + uoff_t high_offset; }; static ssize_t @@ -18,6 +19,7 @@ struct hash_istream *hstream = (struct hash_istream *)stream; const unsigned char *data; size_t size; + uoff_t skip; ssize_t ret; i_stream_seek(stream->parent, stream->parent_start_offset + @@ -27,8 +29,14 @@ if (ret > 0 && hstream->hash_context != NULL) { data = i_stream_get_data(&stream->istream, &size); i_assert((size_t)ret <= size); - hstream->method->loop(hstream->hash_context, - data+(size-ret), ret); + + i_assert(stream->istream.v_offset <= hstream->high_offset); + skip = hstream->high_offset - stream->istream.v_offset; + if (skip < (size_t)size) { + hstream->high_offset += (size-skip); + hstream->method->loop(hstream->hash_context, + data+skip, size-skip); + } } else if (ret < 0) { /* we finished hashing it. don't access it anymore, because the memory pointed by the hash may be freed before the From dovecot at dovecot.org Thu Sep 12 03:52:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 12 Sep 2013 03:52:17 +0300 Subject: dovecot-2.2: doveadm: Don't refer to old doveadm_proxy_port sett... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1db5ede2a1cd changeset: 16727:1db5ede2a1cd user: Timo Sirainen date: Thu Sep 12 03:52:01 2013 +0300 description: doveadm: Don't refer to old doveadm_proxy_port setting name in error message. diffstat: src/doveadm/doveadm-mail-server.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r fe8ac44541ae -r 1db5ede2a1cd src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Thu Sep 12 02:58:51 2013 +0300 +++ b/src/doveadm/doveadm-mail-server.c Thu Sep 12 03:52:01 2013 +0300 @@ -207,7 +207,7 @@ *error_r = fields[0] != NULL ? t_strdup(fields[0]) : "passdb lookup failed"; *error_r = t_strdup_printf("%s: %s (to see if user is proxied, " - "because doveadm_proxy_port is set)", + "because doveadm_port is set)", auth_socket_path, *error_r); } else if (ret == 0) { /* user not found from passdb. it could be in userdb though, From dovecot at dovecot.org Sat Sep 14 00:27:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 14 Sep 2013 00:27:12 +0300 Subject: dovecot-2.2: lib-fs: If backend doesn't implement exists(), emul... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d483e6eb9a2c changeset: 16728:d483e6eb9a2c user: Timo Sirainen date: Sat Sep 14 00:26:45 2013 +0300 description: lib-fs: If backend doesn't implement exists(), emulate it with stat(). diffstat: src/lib-fs/fs-api.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 1db5ede2a1cd -r d483e6eb9a2c src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Sep 12 03:52:01 2013 +0300 +++ b/src/lib-fs/fs-api.c Sat Sep 14 00:26:45 2013 +0300 @@ -536,8 +536,16 @@ int fs_exists(struct fs_file *file) { + struct stat st; int ret; + if (file->fs->v.exists == NULL) { + /* fallback to stat() */ + if (fs_stat(file, &st) == 0) + return 1; + else + return errno == ENOENT ? 0 : -1; + } T_BEGIN { ret = file->fs->v.exists(file); } T_END; From dovecot at dovecot.org Sat Sep 14 00:27:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 14 Sep 2013 00:27:12 +0300 Subject: dovecot-2.2: Comment update. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ae309d070b44 changeset: 16729:ae309d070b44 user: Timo Sirainen date: Sat Sep 14 00:26:58 2013 +0300 description: Comment update. diffstat: src/lib/connection.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r d483e6eb9a2c -r ae309d070b44 src/lib/connection.h --- a/src/lib/connection.h Sat Sep 14 00:26:45 2013 +0300 +++ b/src/lib/connection.h Sat Sep 14 00:26:58 2013 +0300 @@ -36,7 +36,8 @@ void (*client_connected)(struct connection *conn, bool success); /* implement one of the input*() methods. - They return 0 = ok, -1 = error, disconnect the client */ + They return 1 = ok, continue. 0 = ok, but stop processing more + lines, -1 = error, disconnect the client. */ void (*input)(struct connection *conn); int (*input_line)(struct connection *conn, const char *line); int (*input_args)(struct connection *conn, const char *const *args); From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: http-client: Fixed pipelining when payloa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7b775a06c38f changeset: 16730:7b775a06c38f user: Stephan Bosch date: Sun Sep 15 03:27:31 2013 +0300 description: lib-http: http-client: Fixed pipelining when payload synchronizatio (100-continue) is used. Forgot to lock the connection output, which meant that new requests would enqueued for the connection while waiting for 100 Continue. This would cause an assert failure. diffstat: src/lib-http/http-client-connection.c | 5 ++++- src/lib-http/http-client-request.c | 1 + 2 files changed, 5 insertions(+), 1 deletions(-) diffs (38 lines): diff -r ae309d070b44 -r 7b775a06c38f src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Sep 14 00:26:58 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:27:31 2013 +0300 @@ -552,7 +552,7 @@ if (conn->payload_continue) { http_client_connection_debug(conn, "Got 100-continue response after timeout"); - return; + continue; } conn->peer->no_payload_sync = FALSE; conn->peer->seen_100_response = TRUE; @@ -583,11 +583,14 @@ http_client_request_unref(&req); conn->close_indicated = response->connection_close; + if (req->payload_sync && !conn->payload_continue) + conn->output_locked = FALSE; if (!aborted) { if (response->status == 417 && req->payload_sync) { /* drop Expect: continue */ req->payload_sync = FALSE; + conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; http_client_request_retry(req, response->status, response->reason); return; diff -r ae309d070b44 -r 7b775a06c38f src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Sep 14 00:26:58 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:27:31 2013 +0300 @@ -415,6 +415,7 @@ ret = -1; } else { http_client_request_debug(req, "Waiting for 100-continue"); + conn->output_locked = TRUE; } } else { req->state = HTTP_REQUEST_STATE_WAITING; From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: http-client: Connection was using wrong r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/abd565369b8f changeset: 16731:abd565369b8f user: Stephan Bosch date: Sun Sep 15 03:28:21 2013 +0300 description: lib-http: http-client: Connection was using wrong request index in request_wait_list to continue sending outgoing payload. diffstat: src/lib-http/http-client-connection.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 7b775a06c38f -r abd565369b8f src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:27:31 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:28:21 2013 +0300 @@ -241,7 +241,9 @@ http_client_connection_debug(conn, "Expected 100-continue response timed out; sending payload anyway"); - req_idx = array_idx(&conn->request_wait_list, 0); + i_assert(array_count(&conn->request_wait_list) > 0); + req_idx = array_idx(&conn->request_wait_list, + array_count(&conn->request_wait_list)-1); req = req_idx[0]; conn->payload_continue = TRUE; @@ -673,7 +675,8 @@ } if (array_count(&conn->request_wait_list) > 0 && conn->output_locked) { - req_idx = array_idx(&conn->request_wait_list, 0); + req_idx = array_idx(&conn->request_wait_list, + array_count(&conn->request_wait_list)-1); req = req_idx[0]; if (!req->payload_sync || conn->payload_continue) { From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: http-client: Fixed leak of ostream when r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0bcbc30b81b7 changeset: 16732:0bcbc30b81b7 user: Stephan Bosch date: Sun Sep 15 03:29:03 2013 +0300 description: lib-http: http-client: Fixed leak of ostream when request was resubmitted upon 417 response regarding 100-continue. This would only occur when the server refuses Expect: 100-continue with a 417 response. The payload_output stream would be overwritten without being freed first. diffstat: src/lib-http/http-client-request.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (33 lines): diff -r abd565369b8f -r 0bcbc30b81b7 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Sep 15 03:28:21 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:29:03 2013 +0300 @@ -364,6 +364,7 @@ int ret = 0; i_assert(!req->conn->output_locked); + i_assert(req->payload_output == NULL); str_append(rtext, req->method); str_append(rtext, " "); @@ -594,6 +595,10 @@ } } + /* drop payload output stream from previous attempt */ + if (req->payload_output != NULL) + o_stream_unref(&req->payload_output); + newport = (url->have_port ? url->port : (url->have_ssl ? 443 : 80)); target = http_url_create_target(url); @@ -663,6 +668,10 @@ } } + /* drop payload output stream from previous attempt */ + if (req->payload_output != NULL) + o_stream_unref(&req->payload_output); + req->conn = NULL; req->peer = NULL; req->state = HTTP_REQUEST_STATE_QUEUED; From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: http-client: Fixed request scheduling and... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bdc5d6dcfc53 changeset: 16733:bdc5d6dcfc53 user: Stephan Bosch date: Sun Sep 15 03:31:28 2013 +0300 description: lib-http: http-client: Fixed request scheduling and connection management. diffstat: src/lib-http/http-client-connection.c | 49 ++-- src/lib-http/http-client-host.c | 159 +++++++++------- src/lib-http/http-client-peer.c | 317 ++++++++++++++++++++++++++------- src/lib-http/http-client-private.h | 38 ++- src/lib-http/http-client-request.c | 2 +- src/lib-http/http-client.c | 5 + src/lib-http/http-client.h | 4 +- 7 files changed, 393 insertions(+), 181 deletions(-) diffs (truncated from 1003 to 300 lines): diff -r 0bcbc30b81b7 -r bdc5d6dcfc53 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:29:03 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:31:28 2013 +0300 @@ -167,8 +167,7 @@ http_client_connection_unref(&conn); } -static void -http_client_connection_check_idle(struct http_client_connection *conn) +void http_client_connection_check_idle(struct http_client_connection *conn) { unsigned int timeout, count; @@ -254,7 +253,7 @@ } } -bool http_client_connection_next_request(struct http_client_connection *conn) +int http_client_connection_next_request(struct http_client_connection *conn) { struct http_client_request *req = NULL; const char *error; @@ -262,17 +261,15 @@ if (!http_client_connection_is_ready(conn)) { http_client_connection_debug(conn, "Not ready for next request"); - return FALSE; + return 0; } /* claim request, but no urgent request can be second in line */ have_pending_requests = array_count(&conn->request_wait_list) > 0 || conn->pending_request != NULL; req = http_client_peer_claim_request(conn->peer, have_pending_requests); - if (req == NULL) { - http_client_connection_check_idle(conn); - return FALSE; - } + if (req == NULL) + return 0; if (conn->to_idle != NULL) timeout_remove(&conn->to_idle); @@ -300,7 +297,7 @@ http_client_connection_abort_temp_error(&conn, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, t_strdup_printf("Failed to send request: %s", error)); - return FALSE; + return -1; } /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21; @@ -321,7 +318,7 @@ http_client_connection_continue_timeout, conn); } - return TRUE; + return 1; } static void http_client_connection_destroy(struct connection *_conn) @@ -595,7 +592,7 @@ conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; http_client_request_retry(req, response->status, response->reason); - return; + } else if (response->status / 100 == 3 && response->status != 304 && response->location != NULL) { /* redirect */ @@ -648,9 +645,13 @@ } if (finished > 0) { + /* connection still alive after (at least one) request; + we can pipeline -> mark for subsequent connections */ + conn->peer->allows_pipelining = TRUE; + /* room for new requests */ - http_client_peer_handle_requests(conn->peer); - http_client_connection_check_idle(conn); + if (http_client_connection_is_ready(conn)) + http_client_peer_trigger_request_handler(conn->peer); } } @@ -688,8 +689,8 @@ } if (!conn->output_locked) { /* room for new requests */ - http_client_peer_handle_requests(conn->peer); - http_client_connection_check_idle(conn); + if (http_client_connection_is_ready(conn)) + http_client_peer_trigger_request_handler(conn->peer); } } } @@ -701,27 +702,28 @@ { struct stat st; + /* connected */ conn->connected = TRUE; - conn->connect_succeeded = TRUE; if (conn->to_connect != NULL && (conn->ssl_iostream == NULL || ssl_iostream_is_handshaked(conn->ssl_iostream))) timeout_remove(&conn->to_connect); + /* indicate connection success */ + conn->connect_succeeded = TRUE; http_client_peer_connection_success(conn->peer); + /* start raw log */ if (conn->client->set.rawlog_dir != NULL && stat(conn->client->set.rawlog_dir, &st) == 0) { iostream_rawlog_create(conn->client->set.rawlog_dir, &conn->conn.input, &conn->conn.output); } + /* start protocol I/O */ conn->http_parser = http_response_parser_init(conn->conn.input); o_stream_set_flush_callback(conn->conn.output, http_client_connection_output, conn); - - /* we never pipeline before the first response */ - (void)http_client_connection_next_request(conn); } static int @@ -872,23 +874,24 @@ { struct http_client_connection *conn; static unsigned int id = 0; + const struct http_client_peer_addr *addr = &peer->addr; conn = i_new(struct http_client_connection, 1); conn->refcount = 1; conn->client = peer->client; + conn->id = id++; conn->peer = peer; i_array_init(&conn->request_wait_list, 16); connection_init_client_ip - (peer->client->conn_list, &conn->conn, &peer->addr.ip, peer->addr.port); + (peer->client->conn_list, &conn->conn, &addr->ip, addr->port); http_client_connection_connect(conn); - conn->id = id++; array_append(&peer->conns, &conn, 1); http_client_connection_debug(conn, - "Connection created (%d parallel connections exist)", - array_count(&peer->conns)); + "Connection created (%d parallel connections exist)%s", + array_count(&peer->conns), (conn->to_input == NULL ? "" : " [broken]")); return conn; } diff -r 0bcbc30b81b7 -r bdc5d6dcfc53 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sun Sep 15 03:29:03 2013 +0300 +++ b/src/lib-http/http-client-host.c Sun Sep 15 03:31:28 2013 +0300 @@ -46,13 +46,13 @@ static struct http_client_host_port * http_client_host_port_find(struct http_client_host *host, - unsigned int port, const char *https_name) + in_port_t port, const char *https_name) { struct http_client_host_port *hport; array_foreach_modifiable(&host->ports, hport) { - if (hport->port == port && - null_strcmp(hport->https_name, https_name) == 0) + if (hport->addr.port == port && + null_strcmp(hport->addr.https_name, https_name) == 0) return hport; } @@ -61,7 +61,7 @@ static struct http_client_host_port * http_client_host_port_init(struct http_client_host *host, - unsigned int port, const char *https_name) + in_port_t port, const char *https_name) { struct http_client_host_port *hport; @@ -69,8 +69,8 @@ if (hport == NULL) { hport = array_append_space(&host->ports); hport->host = host; - hport->port = port; - hport->https_name = i_strdup(https_name); + hport->addr.port = port; + hport->addr.https_name = i_strdup(https_name); hport->ips_connect_idx = 0; i_array_init(&hport->request_queue, 16); } @@ -94,7 +94,9 @@ { http_client_host_port_error (hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted"); - i_free(hport->https_name); + i_free(hport->addr.https_name); + if (array_is_created(&hport->pending_peers)) + array_free(&hport->pending_peers); array_free(&hport->request_queue); } @@ -136,20 +138,25 @@ if (hport->to_connect != NULL) timeout_remove(&hport->to_connect); - if (http_client_hport_is_last_connect_ip(hport)) + if (http_client_hport_is_last_connect_ip(hport)) { + /* no more IPs to try */ return; + } /* if our our previous connection attempt takes longer than the - soft_connect_timeout we start a connection attempt to the next IP in + soft_connect_timeout, we start a connection attempt to the next IP in parallel */ http_client_host_debug(host, "Connection to %s:%u%s is taking a long time; " "starting parallel connection attempt to next IP", - net_ip2addr(&host->ips[hport->ips_connect_idx]), hport->port, - hport->https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", hport->https_name)); + net_ip2addr(&hport->addr.ip), hport->addr.port, + hport->addr.https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", hport->addr.https_name)); + /* next IP */ hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count; + + /* setup connection to new peer (can start new soft timeout) */ http_client_host_port_connection_setup(hport); } @@ -158,59 +165,41 @@ { struct http_client_host *host = hport->host; struct http_client_peer *peer = NULL; - struct http_client_peer_addr addr; - unsigned int msecs; + const struct http_client_peer_addr *addr = &hport->addr; + unsigned int num_requests = array_count(&hport->request_queue); - addr.ip = host->ips[hport->ips_connect_idx]; - addr.port = hport->port; - addr.https_name = hport->https_name; + if (num_requests == 0) + return; - http_client_host_debug(host, "Setting up connection to %s:%u%s", - net_ip2addr(&addr.ip), addr.port, addr.https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", addr.https_name)); + /* update our peer address */ + hport->addr.ip = host->ips[hport->ips_connect_idx]; - peer = http_client_peer_get(host->client, &addr); + http_client_host_debug(host, "Setting up connection to %s:%u%s " + "(%u requests pending)", net_ip2addr(&addr->ip), addr->port, + addr->https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", addr->https_name), num_requests); + + /* create/get peer */ + peer = http_client_peer_get(host->client, addr); http_client_peer_add_host(peer, host); - if (http_client_peer_handle_requests(peer)) - hport->pending_connection_count++; - /* start soft connect time-out (but only if we have another IP left) */ - msecs = host->client->set.soft_connect_timeout_msecs; - if (!http_client_hport_is_last_connect_ip(hport) && msecs > 0 && - hport->to_connect == NULL) { - hport->to_connect = - timeout_add(msecs, http_client_host_port_soft_connect_timeout, hport); - } -} + /* handle requests; creates new connections when needed/possible */ + http_client_peer_trigger_request_handler(peer); -static void -http_client_host_drop_pending_connections(struct http_client_host_port *hport, - const struct http_client_peer_addr *addr) -{ - struct http_client_peer *peer; - struct http_client_connection *const *conns, *conn; - unsigned int i, count; + if (!http_client_peer_is_connected(peer)) { + unsigned int msecs; - for (peer = hport->host->client->peers_list; peer != NULL; peer = peer->next) { - if (http_client_peer_addr_cmp(&peer->addr, addr) == 0) { - /* don't drop any connections to the successfully - connected peer, even if some of the connections - are pending. they may be intended for urgent From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Fixed handling of non-standard CRLF at en... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/819dd674f100 changeset: 16734:819dd674f100 user: Stephan Bosch date: Sun Sep 15 03:31:51 2013 +0300 description: lib-http: Fixed handling of non-standard CRLF at end of request. diffstat: src/lib-http/http-request-parser.c | 32 ++++++++++++++++++++++++-------- 1 files changed, 24 insertions(+), 8 deletions(-) diffs (69 lines): diff -r bdc5d6dcfc53 -r 819dd674f100 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:31:28 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:31:51 2013 +0300 @@ -8,6 +8,7 @@ enum http_request_parser_state { HTTP_REQUEST_PARSE_STATE_INIT = 0, + HTTP_REQUEST_PARSE_STATE_SKIP_LINE, HTTP_REQUEST_PARSE_STATE_METHOD, HTTP_REQUEST_PARSE_STATE_SP1, HTTP_REQUEST_PARSE_STATE_TARGET, @@ -23,6 +24,8 @@ enum http_request_parser_state state; struct http_request request; + + unsigned int skipping_line:1; }; struct http_request_parser *http_request_parser_init(struct istream *input) @@ -105,15 +108,24 @@ switch (parser->state) { case HTTP_REQUEST_PARSE_STATE_INIT: http_request_parser_restart(parser); - parser->state = HTTP_REQUEST_PARSE_STATE_VERSION; + parser->state = HTTP_REQUEST_PARSE_STATE_SKIP_LINE; if (_parser->cur == _parser->end) return 0; + case HTTP_REQUEST_PARSE_STATE_SKIP_LINE: if (*_parser->cur == '\r' || *_parser->cur == '\n') { - /* HTTP/1.0 client sent a CRLF after body. + if (parser->skipping_line) { + /* second extra CRLF; not allowed */ + *error_r = "Empty request line"; + return -1; + } + /* HTTP/1.0 client sent one extra CRLF after body. ignore it. */ + parser->skipping_line = TRUE; parser->state = HTTP_REQUEST_PARSE_STATE_CR; - return http_request_parse(parser, error_r); + break; } + parser->state = HTTP_REQUEST_PARSE_STATE_METHOD; + parser->skipping_line = FALSE; /* fall through */ case HTTP_REQUEST_PARSE_STATE_METHOD: if ((ret=http_request_parse_method(parser)) <= 0) { @@ -184,11 +196,15 @@ return -1; } _parser->cur++; - parser->state = HTTP_REQUEST_PARSE_STATE_HEADER; - return 1; - case HTTP_REQUEST_PARSE_STATE_HEADER: - default: - i_unreached(); + if (!parser->skipping_line) { + parser->state = HTTP_REQUEST_PARSE_STATE_HEADER; + return 1; + } + parser->state = HTTP_REQUEST_PARSE_STATE_INIT; + break; + case HTTP_REQUEST_PARSE_STATE_HEADER: + default: + i_unreached(); } } From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Removed useless prev,next fields from str... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9af78d53e22f changeset: 16735:9af78d53e22f user: Stephan Bosch date: Sun Sep 15 03:32:11 2013 +0300 description: lib-http: Removed useless prev,next fields from struct http_client_request. diffstat: src/lib-http/http-client-private.h | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 819dd674f100 -r 9af78d53e22f src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sun Sep 15 03:31:51 2013 +0300 +++ b/src/lib-http/http-client-private.h Sun Sep 15 03:32:11 2013 +0300 @@ -39,8 +39,6 @@ pool_t pool; unsigned int refcount; - struct http_client_request *prev, *next; - const char *method, *hostname, *target; in_port_t port; From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Fixed client connection and peer log labe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb6506d2d084 changeset: 16736:bb6506d2d084 user: Stephan Bosch date: Sun Sep 15 03:33:04 2013 +0300 description: lib-http: Fixed client connection and peer log labels to show proper IPv6 peers. The labels did not put IPv6 addresses in square brackets. diffstat: src/lib-http/http-client-host.c | 29 ++++++++++++++--------------- src/lib-http/http-client-peer.c | 5 ++--- src/lib-http/http-client-private.h | 18 ++++++++++++++++-- 3 files changed, 32 insertions(+), 20 deletions(-) diffs (125 lines): diff -r 9af78d53e22f -r bb6506d2d084 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sun Sep 15 03:32:11 2013 +0300 +++ b/src/lib-http/http-client-host.c Sun Sep 15 03:33:04 2013 +0300 @@ -134,6 +134,7 @@ http_client_host_port_soft_connect_timeout(struct http_client_host_port *hport) { struct http_client_host *host = hport->host; + const struct http_client_peer_addr *addr = &hport->addr; if (hport->to_connect != NULL) timeout_remove(&hport->to_connect); @@ -146,12 +147,10 @@ /* if our our previous connection attempt takes longer than the soft_connect_timeout, we start a connection attempt to the next IP in parallel */ - - http_client_host_debug(host, "Connection to %s:%u%s is taking a long time; " + http_client_host_debug(host, "Connection to %s%s is taking a long time; " "starting parallel connection attempt to next IP", - net_ip2addr(&hport->addr.ip), hport->addr.port, - hport->addr.https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", hport->addr.https_name)); + http_client_peer_addr2str(addr), addr->https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", addr->https_name)); /* next IP */ hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count; @@ -174,10 +173,10 @@ /* update our peer address */ hport->addr.ip = host->ips[hport->ips_connect_idx]; - http_client_host_debug(host, "Setting up connection to %s:%u%s " - "(%u requests pending)", net_ip2addr(&addr->ip), addr->port, - addr->https_name == NULL ? "" : - t_strdup_printf(" (SSL=%s)", addr->https_name), num_requests); + http_client_host_debug(host, "Setting up connection to %s%s " + "(%u requests pending)", http_client_peer_addr2str(addr), + (addr->https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", addr->https_name)), num_requests); /* create/get peer */ peer = http_client_peer_get(host->client, addr); @@ -307,8 +306,8 @@ { struct http_client_host_port *hport; - http_client_host_debug(host, "Successfully connected to %s:%u", - net_ip2addr(&addr->ip), addr->port); + http_client_host_debug(host, "Successfully connected to %s", + http_client_peer_addr2str(addr)); hport = http_client_host_port_find(host, addr->port, addr->https_name); if (hport == NULL) @@ -322,8 +321,8 @@ { struct http_client_host_port *hport; - http_client_host_debug(host, "Failed to connect to %s:%u: %s", - net_ip2addr(&addr->ip), addr->port, reason); + http_client_host_debug(host, "Failed to connect to %s: %s", + http_client_peer_addr2str(addr), reason); hport = http_client_host_port_find(host, addr->port, addr->https_name); if (hport == NULL) @@ -513,8 +512,8 @@ array_delete(&hport->request_queue, i, 1); http_client_host_debug(host, - "Connection to peer %s:%u claimed request %s %s", - net_ip2addr(&addr->ip), addr->port, http_client_request_label(req), + "Connection to peer %s claimed request %s %s", + http_client_peer_addr2str(addr), http_client_request_label(req), (req->urgent ? "(urgent)" : "")); return req; diff -r 9af78d53e22f -r bb6506d2d084 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Sun Sep 15 03:32:11 2013 +0300 +++ b/src/lib-http/http-client-peer.c Sun Sep 15 03:33:04 2013 +0300 @@ -29,9 +29,8 @@ if (peer->client->set.debug) { va_start(args, format); - i_debug("http-client: peer %s:%u: %s", - net_ip2addr(&peer->addr.ip), peer->addr.port, - t_strdup_vprintf(format, args)); + i_debug("http-client: peer %s: %s", + http_client_peer_label(peer), t_strdup_vprintf(format, args)); va_end(args); } } diff -r 9af78d53e22f -r bb6506d2d084 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sun Sep 15 03:32:11 2013 +0300 +++ b/src/lib-http/http-client-private.h Sun Sep 15 03:33:04 2013 +0300 @@ -190,6 +190,14 @@ }; static inline const char * +http_client_peer_addr2str(const struct http_client_peer_addr *addr) +{ + if (addr->ip.family == AF_INET6) + return t_strdup_printf("[%s]:%u", net_ip2addr(&addr->ip), addr->port); + return t_strdup_printf("%s:%u", net_ip2addr(&addr->ip), addr->port); +} + +static inline const char * http_client_request_label(struct http_client_request *req) { return t_strdup_printf("[%s http%s://%s:%d%s]", @@ -199,8 +207,14 @@ static inline const char * http_client_connection_label(struct http_client_connection *conn) { - return t_strdup_printf("%s:%u [%d]", - net_ip2addr(&conn->conn.ip), conn->conn.port, conn->id); + return t_strdup_printf("%s [%d]", + http_client_peer_addr2str(&conn->peer->addr), conn->id); +} + +static inline const char * +http_client_peer_label(struct http_client_peer *peer) +{ + return http_client_peer_addr2str(&peer->addr); } int http_client_init_ssl_ctx(struct http_client *client, const char **error_r); From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: http-client: Requests now automatically g... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e83f3d16a31d changeset: 16737:e83f3d16a31d user: Stephan Bosch date: Sun Sep 15 03:33:44 2013 +0300 description: lib-http: http-client: Requests now automatically generate a Date header. The used date value is normally the submission time of the request, but it can be set explicitly. diffstat: src/lib-http/http-client-private.h | 2 ++ src/lib-http/http-client-request.c | 22 ++++++++++++++++++++-- src/lib-http/http-client.h | 3 +++ 3 files changed, 25 insertions(+), 2 deletions(-) diffs (93 lines): diff -r bb6506d2d084 -r e83f3d16a31d src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sun Sep 15 03:33:04 2013 +0300 +++ b/src/lib-http/http-client-private.h Sun Sep 15 03:33:44 2013 +0300 @@ -48,6 +48,8 @@ struct http_client_connection *conn; string_t *headers; + time_t date; + struct istream *payload_input; uoff_t payload_size, payload_offset; struct ostream *payload_output; diff -r bb6506d2d084 -r e83f3d16a31d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Sep 15 03:33:04 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:33:44 2013 +0300 @@ -8,6 +8,7 @@ #include "istream.h" #include "ostream.h" #include "http-url.h" +#include "http-date.h" #include "http-response-parser.h" #include "http-transfer.h" @@ -72,6 +73,7 @@ req->callback = callback; req->context = context; req->headers = str_new(default_pool, 256); + req->date = (time_t)-1; req->state = HTTP_REQUEST_STATE_NEW; return req; @@ -149,9 +151,20 @@ const char *key, const char *value) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); + /* don't allow setting Date header directly; + this is ignored for now for backwards compatibility */ + if (strcasecmp(key, "Date") == 0) + return; str_printfa(req->headers, "%s: %s\r\n", key, value); } +void http_client_request_set_date(struct http_client_request *req, + time_t date) +{ + i_assert(req->state == HTTP_REQUEST_STATE_NEW); + req->date = date; +} + void http_client_request_set_payload(struct http_client_request *req, struct istream *input, bool sync) { @@ -188,6 +201,10 @@ struct http_client_host *host; i_assert(req->state == HTTP_REQUEST_STATE_NEW); + + /* use submission date if no date is set explicitly */ + if (req->date == (time_t)-1) + req->date = ioloop_time; host = http_client_host_get(req->client, req->hostname); req->state = HTTP_REQUEST_STATE_QUEUED; @@ -369,13 +386,14 @@ str_append(rtext, req->method); str_append(rtext, " "); str_append(rtext, req->target); - str_append(rtext, " HTTP/1.1\r\n"); - str_append(rtext, "Host: "); + str_append(rtext, " HTTP/1.1\r\nHost: "); str_append(rtext, req->hostname); if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) || (req->ssl && req->port != HTTPS_DEFAULT_PORT)) { str_printfa(rtext, ":%u", req->port); } + str_append(rtext, "\r\nDate: "); + str_append(rtext, http_date_create(req->date)); str_append(rtext, "\r\n"); if (req->payload_sync) { str_append(rtext, "Expect: 100-continue\r\n"); diff -r bb6506d2d084 -r e83f3d16a31d src/lib-http/http-client.h --- a/src/lib-http/http-client.h Sun Sep 15 03:33:04 2013 +0300 +++ b/src/lib-http/http-client.h Sun Sep 15 03:33:44 2013 +0300 @@ -98,6 +98,9 @@ void http_client_request_add_header(struct http_client_request *req, const char *key, const char *value); +void http_client_request_set_date(struct http_client_request *req, + time_t date); + void http_client_request_set_payload(struct http_client_request *req, struct istream *input, bool sync); From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Added support for asynchronous payload fo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a6ed95a30cb1 changeset: 16739:a6ed95a30cb1 user: Stephan Bosch date: Sun Sep 15 03:35:04 2013 +0300 description: lib-http: Added support for asynchronous payload for requests. This means that the payload stream passed to the request can be a non-blocking socket stream from some other connection (e.g. proxy client connection). diffstat: src/lib-http/http-client-connection.c | 6 +++++- src/lib-http/http-client-private.h | 2 ++ src/lib-http/http-client-request.c | 32 ++++++++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 5 deletions(-) diffs (133 lines): diff -r 3bd334529536 -r a6ed95a30cb1 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:34:06 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:35:04 2013 +0300 @@ -655,7 +655,7 @@ } } -static int http_client_connection_output(struct http_client_connection *conn) +int http_client_connection_output(struct http_client_connection *conn) { struct http_client_request *const *req_idx, *req; struct ostream *output = conn->conn.output; @@ -944,6 +944,8 @@ ssl_iostream_unref(&conn->ssl_iostream); connection_deinit(&conn->conn); + if (conn->io_req_payload != NULL) + io_remove(&conn->io_req_payload); if (conn->to_requests != NULL) timeout_remove(&conn->to_requests); if (conn->to_connect != NULL) @@ -972,6 +974,8 @@ void http_client_connection_switch_ioloop(struct http_client_connection *conn) { + if (conn->io_req_payload != NULL) + conn->io_req_payload = io_loop_move_io(&conn->io_req_payload); if (conn->to_requests != NULL) conn->to_requests = io_loop_move_timeout(&conn->to_requests); if (conn->to_connect != NULL) diff -r 3bd334529536 -r a6ed95a30cb1 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sun Sep 15 03:34:06 2013 +0300 +++ b/src/lib-http/http-client-private.h Sun Sep 15 03:35:04 2013 +0300 @@ -161,6 +161,7 @@ struct http_client_request *pending_request; struct istream *incoming_payload; + struct io *io_req_payload; /* requests that have been sent, waiting for response */ ARRAY_TYPE(http_client_request) request_wait_list; @@ -244,6 +245,7 @@ http_client_connection_create(struct http_client_peer *peer); void http_client_connection_ref(struct http_client_connection *conn); void http_client_connection_unref(struct http_client_connection **_conn); +int http_client_connection_output(struct http_client_connection *conn); unsigned int http_client_connection_count_pending(struct http_client_connection *conn); bool http_client_connection_is_ready(struct http_client_connection *conn); diff -r 3bd334529536 -r a6ed95a30cb1 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Sep 15 03:34:06 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:35:04 2013 +0300 @@ -232,7 +232,8 @@ } req->state = HTTP_REQUEST_STATE_WAITING; req->conn->output_locked = FALSE; - http_client_request_debug(req, "Sent all payload"); + + http_client_request_debug(req, "Finished sending payload"); } static int @@ -321,16 +322,30 @@ return http_client_request_continue_payload(_req, NULL, 0); } +static void http_client_request_payload_input(struct http_client_request *req) +{ + struct http_client_connection *conn = req->conn; + + if (conn->io_req_payload != NULL) + io_remove(&conn->io_req_payload); + + (void)http_client_connection_output(conn); +} + int http_client_request_send_more(struct http_client_request *req, const char **error_r) { struct http_client_connection *conn = req->conn; struct ostream *output = req->payload_output; off_t ret; + int fd; i_assert(req->payload_input != NULL); i_assert(req->payload_output != NULL); + if (conn->io_req_payload != NULL) + io_remove(&conn->io_req_payload); + /* chunked ostream needs to write to the parent stream's buffer */ o_stream_set_max_buffer_size(output, IO_BLOCK_SIZE); ret = o_stream_send_istream(output, req->payload_input); @@ -340,15 +355,17 @@ errno = req->payload_input->stream_errno; *error_r = t_strdup_printf("read(%s) failed: %m", i_stream_get_name(req->payload_input)); + ret = -1; } else if (output->stream_errno != 0) { errno = output->stream_errno; *error_r = t_strdup_printf("write(%s) failed: %m", o_stream_get_name(output)); + ret = -1; } else { i_assert(ret >= 0); } - if (!i_stream_have_bytes_left(req->payload_input)) { + if (ret < 0 || i_stream_is_eof(req->payload_input)) { if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { i_error("stream input size changed"); //FIXME @@ -362,11 +379,18 @@ } else { http_client_request_finish_payload_out(req); } - - } else { + } else if (i_stream_get_data_size(req->payload_input) > 0) { + /* output is blocking */ conn->output_locked = TRUE; o_stream_set_flush_pending(output, TRUE); http_client_request_debug(req, "Partially sent payload"); + } else { + /* input is blocking */ + fd = i_stream_get_fd(req->payload_input); + conn->output_locked = TRUE; + i_assert(fd >= 0); + conn->io_req_payload = io_add + (fd, IO_READ, http_client_request_payload_input, req); } return ret < 0 ? -1 : 0; } From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: uri-util: Added support for parsing bare authority ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3bd334529536 changeset: 16738:3bd334529536 user: Stephan Bosch date: Sun Sep 15 03:34:06 2013 +0300 description: uri-util: Added support for parsing bare authority URI component (for use in HTTP). diffstat: src/lib-http/http-url.c | 2 +- src/lib-imap/imap-url.c | 2 +- src/lib/uri-util.c | 24 +++++++++++++++--------- src/lib/uri-util.h | 5 ++++- 4 files changed, 21 insertions(+), 12 deletions(-) diffs (87 lines): diff -r e83f3d16a31d -r 3bd334529536 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Sun Sep 15 03:33:44 2013 +0300 +++ b/src/lib-http/http-url.c Sun Sep 15 03:34:06 2013 +0300 @@ -69,7 +69,7 @@ } /* "//" host [ ":" port ] */ - if ((ret = uri_parse_authority(parser, &auth)) < 0) + if ((ret = uri_parse_slashslash_authority(parser, &auth)) < 0) return FALSE; if (ret > 0) { if (auth.enc_userinfo != NULL) { diff -r e83f3d16a31d -r 3bd334529536 src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Sun Sep 15 03:33:44 2013 +0300 +++ b/src/lib-imap/imap-url.c Sun Sep 15 03:34:06 2013 +0300 @@ -221,7 +221,7 @@ */ /* "//" iserver */ - if ((ret = uri_parse_authority(parser, &auth)) <= 0) + if ((ret = uri_parse_slashslash_authority(parser, &auth)) <= 0) return ret; /* iuserinfo = enc-user [iauth] / [enc-user] iauth */ diff -r e83f3d16a31d -r 3bd334529536 src/lib/uri-util.c --- a/src/lib/uri-util.c Sun Sep 15 03:33:44 2013 +0300 +++ b/src/lib/uri-util.c Sun Sep 15 03:34:06 2013 +0300 @@ -509,22 +509,16 @@ return 0; } -int uri_parse_authority(struct uri_parser *parser, struct uri_authority *auth) +int uri_parse_authority(struct uri_parser *parser, + struct uri_authority *auth) { const unsigned char *p; int ret; - /* hier-part = "//" authority {...} - * relative-part = "//" authority {...} + /* * authority = [ userinfo "@" ] host [ ":" port ] */ - /* Parse "//" as part of authority */ - if ((parser->end - parser->cur) <= 2 || parser->cur[0] != '/' || - parser->cur[1] != '/') - return 0; - parser->cur += 2; - if (auth != NULL) memset(auth, 0, sizeof(*auth)); @@ -566,6 +560,18 @@ return 1; } +int uri_parse_slashslash_authority(struct uri_parser *parser, + struct uri_authority *auth) +{ + /* "//" authority */ + + if ((parser->end - parser->cur) <= 2 || parser->cur[0] != '/' || + parser->cur[1] != '/') + return 0; + + return uri_parse_authority(parser, auth); +} + int uri_parse_path_segment(struct uri_parser *parser, const char **segment_r) { const unsigned char *p = parser->cur; diff -r e83f3d16a31d -r 3bd334529536 src/lib/uri-util.h --- a/src/lib/uri-util.h Sun Sep 15 03:33:44 2013 +0300 +++ b/src/lib/uri-util.h Sun Sep 15 03:34:06 2013 +0300 @@ -35,7 +35,10 @@ int uri_cut_scheme(const char **uri_p, const char **scheme_r); int uri_parse_scheme(struct uri_parser *parser, const char **scheme_r); -int uri_parse_authority(struct uri_parser *parser, struct uri_authority *auth); +int uri_parse_authority(struct uri_parser *parser, + struct uri_authority *auth); +int uri_parse_slashslash_authority(struct uri_parser *parser, + struct uri_authority *auth); int uri_parse_path_segment(struct uri_parser *parser, const char **segment_r); int uri_parse_path(struct uri_parser *parser, int *relative_r, From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Adjusted response and request parsers to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/adb4d013073d changeset: 16740:adb4d013073d user: Stephan Bosch date: Sun Sep 15 03:36:18 2013 +0300 description: lib-http: Adjusted response and request parsers to accept a request/response object to fill with data, rather than have it return one. diffstat: src/lib-http/http-client-connection.c | 24 +++++++++--------- src/lib-http/http-request-parser.c | 29 ++++++++++++---------- src/lib-http/http-request-parser.h | 2 +- src/lib-http/http-response-parser.c | 40 +++++++++++++++++-------------- src/lib-http/http-response-parser.h | 2 +- src/lib-http/test-http-response-parser.c | 11 ++++---- src/lib-http/test-http-server.c | 9 ++++--- 7 files changed, 62 insertions(+), 55 deletions(-) diffs (truncated from 345 to 300 lines): diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:36:18 2013 +0300 @@ -488,7 +488,7 @@ { struct http_client_connection *conn = (struct http_client_connection *)_conn; - struct http_response *response; + struct http_response response; struct http_client_request *const *req_idx; struct http_client_request *req = NULL; int finished = 0, ret; @@ -547,7 +547,7 @@ (Continue) status message. Unexpected 1xx status responses MAY be ignored by a user agent. */ - if (req->payload_sync && response->status == 100) { + if (req->payload_sync && response.status == 100) { if (conn->payload_continue) { http_client_connection_debug(conn, "Got 100-continue response after timeout"); @@ -564,16 +564,16 @@ t_strdup_printf("Failed to send request: %s", error)); } return; - } else if (response->status / 100 == 1) { + } else if (response.status / 100 == 1) { /* ignore them for now */ http_client_connection_debug(conn, - "Got unexpected %u response; ignoring", response->status); + "Got unexpected %u response; ignoring", response.status); continue; } http_client_connection_debug(conn, "Got %u response for request %s", - response->status, http_client_request_label(req)); + response.status, http_client_request_label(req)); /* remove request from queue */ array_delete(&conn->request_wait_list, 0, 1); @@ -581,25 +581,25 @@ i_assert(req->refcount > 1 || aborted); http_client_request_unref(&req); - conn->close_indicated = response->connection_close; + conn->close_indicated = response.connection_close; if (req->payload_sync && !conn->payload_continue) conn->output_locked = FALSE; if (!aborted) { - if (response->status == 417 && req->payload_sync) { + if (response.status == 417 && req->payload_sync) { /* drop Expect: continue */ req->payload_sync = FALSE; conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; - http_client_request_retry(req, response->status, response->reason); + http_client_request_retry(req, response.status, response.reason); - } else if (response->status / 100 == 3 && response->status != 304 && - response->location != NULL) { + } else if (response.status / 100 == 3 && response.status != 304 && + response.location != NULL) { /* redirect */ - http_client_request_redirect(req, response->status, response->location); + http_client_request_redirect(req, response.status, response.location); } else { /* response for application */ - if (!http_client_connection_return_response(conn, req, response)) + if (!http_client_connection_return_response(conn, req, &response)) return; } diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:36:18 2013 +0300 @@ -23,7 +23,8 @@ struct http_message_parser parser; enum http_request_parser_state state; - struct http_request request; + const char *request_method; + const char *request_target; unsigned int skipping_line:1; }; @@ -49,7 +50,8 @@ http_request_parser_restart(struct http_request_parser *parser) { http_message_parser_restart(&parser->parser); - memset(&parser->request, 0, sizeof(parser->request)); + parser->request_method = NULL; + parser->request_target = NULL; } static int http_request_parse_method(struct http_request_parser *parser) @@ -63,7 +65,7 @@ if (p == parser->parser.end) return 0; - parser->request.method = + parser->request_method = p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); parser->parser.cur = p; return 1; @@ -82,7 +84,7 @@ if (p == parser->parser.end) return 0; - parser->request.target = + parser->request_target = p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); parser->parser.cur = p; return 1; @@ -246,7 +248,7 @@ } int http_request_parse_next(struct http_request_parser *parser, - struct http_request **request_r, + struct http_request *request, const char **error_r) { int ret; @@ -271,13 +273,14 @@ return -1; parser->state = HTTP_REQUEST_PARSE_STATE_INIT; - parser->request.version_major = parser->parser.msg.version_major; - parser->request.version_minor = parser->parser.msg.version_minor; - parser->request.date = parser->parser.msg.date; - parser->request.payload = parser->parser.payload; - parser->request.headers = parser->parser.msg.headers; - parser->request.connection_close = parser->parser.msg.connection_close; - - *request_r = &parser->request; + memset(request, 0, sizeof(*request)); + request->method = parser->request_method; + request->target = parser->request_target; + request->version_major = parser->parser.msg.version_major; + request->version_minor = parser->parser.msg.version_minor; + request->date = parser->parser.msg.date; + request->payload = parser->parser.payload; + request->headers = parser->parser.msg.headers; + request->connection_close = parser->parser.msg.connection_close; return 1; } diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:36:18 2013 +0300 @@ -23,7 +23,7 @@ void http_request_parser_deinit(struct http_request_parser **_parser); int http_request_parse_next(struct http_request_parser *parser, - struct http_request **request_r, + struct http_request *request, const char **error_r); #endif diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/http-response-parser.c Sun Sep 15 03:36:18 2013 +0300 @@ -24,7 +24,8 @@ struct http_message_parser parser; enum http_response_parser_state state; - struct http_response response; + unsigned int response_status; + const char *response_reason; }; struct http_response_parser *http_response_parser_init(struct istream *input) @@ -48,7 +49,8 @@ http_response_parser_restart(struct http_response_parser *parser) { http_message_parser_restart(&parser->parser); - memset(&parser->response, 0, sizeof(parser->response)); + parser->response_status = 0; + parser->response_reason = NULL; } static int http_response_parse_status(struct http_response_parser *parser) @@ -62,7 +64,7 @@ return 0; if (!i_isdigit(p[0]) || !i_isdigit(p[1]) || !i_isdigit(p[2])) return -1; - parser->response.status = + parser->response_status = (p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0'); parser->parser.cur += 3; return 1; @@ -74,12 +76,13 @@ /* reason-phrase = *( HTAB / SP / VCHAR / obs-text ) */ + // FIXME: limit length while (p < parser->parser.end && http_char_is_text(*p)) p++; if (p == parser->parser.end) return 0; - parser->response.reason = + parser->response_reason = p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); parser->parser.cur = p; return 1; @@ -223,7 +226,7 @@ } int http_response_parse_next(struct http_response_parser *parser, - bool no_payload, struct http_response **response_r, + bool no_payload, struct http_response *response, const char **error_r) { int ret; @@ -251,11 +254,11 @@ A server MUST NOT send a Content-Length header field in any response with a status code of 1xx (Informational) or 204 (No Content). [...] */ - if ((parser->response.status / 100 == 1 || parser->response.status == 204) && + if ((parser->response_status / 100 == 1 || parser->response_status == 204) && parser->parser.msg.content_length > 0) { *error_r = t_strdup_printf( "Unexpected Content-Length header field for %u response " - "(length=%"PRIuUOFF_T")", parser->response.status, + "(length=%"PRIuUOFF_T")", parser->response_status, parser->parser.msg.content_length); return -1; } @@ -269,8 +272,8 @@ header fields, regardless of the header fields present in the message, and thus cannot contain a message body. */ - if (parser->response.status / 100 == 1 || parser->response.status == 204 - || parser->response.status == 304) { // HEAD is handled in caller + if (parser->response_status / 100 == 1 || parser->response_status == 204 + || parser->response_status == 304) { // HEAD is handled in caller no_payload = TRUE; } @@ -281,14 +284,15 @@ } parser->state = HTTP_RESPONSE_PARSE_STATE_INIT; - parser->response.version_major = parser->parser.msg.version_major; - parser->response.version_minor = parser->parser.msg.version_minor; - parser->response.location = parser->parser.msg.location; - parser->response.date = parser->parser.msg.date; - parser->response.payload = parser->parser.payload; - parser->response.headers = parser->parser.msg.headers; - parser->response.connection_close = parser->parser.msg.connection_close; - - *response_r = &parser->response; + memset(response, 0, sizeof(*response)); + response->status = parser->response_status; + response->reason = parser->response_reason; + response->version_major = parser->parser.msg.version_major; + response->version_minor = parser->parser.msg.version_minor; + response->location = parser->parser.msg.location; + response->date = parser->parser.msg.date; + response->payload = parser->parser.payload; + response->headers = parser->parser.msg.headers; + response->connection_close = parser->parser.msg.connection_close; return 1; } diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/http-response-parser.h --- a/src/lib-http/http-response-parser.h Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/http-response-parser.h Sun Sep 15 03:36:18 2013 +0300 @@ -10,7 +10,7 @@ void http_response_parser_deinit(struct http_response_parser **_parser); int http_response_parse_next(struct http_response_parser *parser, - bool no_payload, struct http_response **response_r, + bool no_payload, struct http_response *response, const char **error_r); #endif diff -r a6ed95a30cb1 -r adb4d013073d src/lib-http/test-http-response-parser.c --- a/src/lib-http/test-http-response-parser.c Sun Sep 15 03:35:04 2013 +0300 +++ b/src/lib-http/test-http-response-parser.c Sun Sep 15 03:36:18 2013 +0300 @@ -99,7 +99,7 @@ struct ostream *output; const struct http_response_parse_test *test; struct http_response_parser *parser; - struct http_response *response = NULL; + struct http_response response; const char *response_text, *payload, *error; unsigned int pos, response_text_len; int ret = 0; @@ -119,11 +119,11 @@ } test_istream_set_size(input, response_text_len); while (ret > 0) { - if (response->payload != NULL) { + if (response.payload != NULL) { buffer_set_used_size(payload_buffer, 0); output = o_stream_create_buffer(payload_buffer); test_out("payload receive", - o_stream_send_istream(output, response->payload)); + o_stream_send_istream(output, response.payload)); From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Adjusted message parser to accept pool fr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b172c130df9b changeset: 16741:b172c130df9b user: Stephan Bosch date: Sun Sep 15 03:36:44 2013 +0300 description: lib-http: Adjusted message parser to accept pool from caller. diffstat: src/lib-http/http-message-parser.c | 24 +++++++++++++++--------- src/lib-http/http-message-parser.h | 5 ++++- src/lib-http/http-request-parser.c | 21 +++++++++++---------- src/lib-http/http-request-parser.h | 2 +- src/lib-http/http-response-parser.c | 4 ++-- src/lib-http/test-http-server.c | 2 +- 6 files changed, 34 insertions(+), 24 deletions(-) diffs (209 lines): diff -r adb4d013073d -r b172c130df9b src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:36:44 2013 +0300 @@ -22,13 +22,14 @@ { if (parser->header_parser != NULL) http_header_parser_deinit(&parser->header_parser); - if (parser->msg_pool != NULL) - pool_unref(&parser->msg_pool); + if (parser->msg.pool != NULL) + pool_unref(&parser->msg.pool); if (parser->payload != NULL) i_stream_unref(&parser->payload); } -void http_message_parser_restart(struct http_message_parser *parser) +void http_message_parser_restart(struct http_message_parser *parser, + pool_t pool) { i_assert(parser->payload == NULL); @@ -37,12 +38,17 @@ else http_header_parser_reset(parser->header_parser); - if (parser->msg_pool != NULL) - pool_unref(&parser->msg_pool); - parser->msg_pool = pool_alloconly_create("http_message", 4096); + if (parser->msg.pool != NULL) + pool_unref(&parser->msg.pool); memset(&parser->msg, 0, sizeof(parser->msg)); + if (pool == NULL) { + parser->msg.pool = pool_alloconly_create("http_message", 4096); + } else { + parser->msg.pool = pool; + pool_ref(pool); + } parser->msg.date = (time_t)-1; - p_array_init(&parser->msg.headers, parser->msg_pool, 32); + p_array_init(&parser->msg.headers, parser->msg.pool, 32); } int http_message_parse_version(struct http_message_parser *parser) @@ -95,8 +101,8 @@ void *value; hdr = array_append_space(&parser->msg.headers); - hdr->key = p_strdup(parser->msg_pool, name); - hdr->value = value = p_malloc(parser->msg_pool, size+1); + hdr->key = p_strdup(parser->msg.pool, name); + hdr->value = value = p_malloc(parser->msg.pool, size+1); memcpy(value, data, size); hdr->size = size; diff -r adb4d013073d -r b172c130df9b src/lib-http/http-message-parser.h --- a/src/lib-http/http-message-parser.h Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/http-message-parser.h Sun Sep 15 03:36:44 2013 +0300 @@ -4,6 +4,8 @@ #include "http-response.h" struct http_message { + pool_t pool; + unsigned int version_major; unsigned int version_minor; @@ -32,7 +34,8 @@ void http_message_parser_init(struct http_message_parser *parser, struct istream *input); void http_message_parser_deinit(struct http_message_parser *parser); -void http_message_parser_restart(struct http_message_parser *parser); +void http_message_parser_restart(struct http_message_parser *parser, + pool_t pool); int http_message_parse_finish_payload(struct http_message_parser *parser, const char **error_r); diff -r adb4d013073d -r b172c130df9b src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:36:44 2013 +0300 @@ -47,9 +47,10 @@ } static void -http_request_parser_restart(struct http_request_parser *parser) +http_request_parser_restart(struct http_request_parser *parser, + pool_t pool) { - http_message_parser_restart(&parser->parser); + http_message_parser_restart(&parser->parser, pool); parser->request_method = NULL; parser->request_target = NULL; } @@ -66,7 +67,7 @@ if (p == parser->parser.end) return 0; parser->request_method = - p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); + p_strdup_until(parser->parser.msg.pool, parser->parser.cur, p); parser->parser.cur = p; return 1; } @@ -85,7 +86,7 @@ if (p == parser->parser.end) return 0; parser->request_target = - p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); + p_strdup_until(parser->parser.msg.pool, parser->parser.cur, p); parser->parser.cur = p; return 1; } @@ -98,7 +99,7 @@ } static int http_request_parse(struct http_request_parser *parser, - const char **error_r) + pool_t pool, const char **error_r) { struct http_message_parser *_parser = &parser->parser; int ret; @@ -109,7 +110,7 @@ for (;;) { switch (parser->state) { case HTTP_REQUEST_PARSE_STATE_INIT: - http_request_parser_restart(parser); + http_request_parser_restart(parser, pool); parser->state = HTTP_REQUEST_PARSE_STATE_SKIP_LINE; if (_parser->cur == _parser->end) return 0; @@ -215,7 +216,7 @@ } static int http_request_parse_request_line(struct http_request_parser *parser, - const char **error_r) + pool_t pool, const char **error_r) { struct http_message_parser *_parser = &parser->parser; const unsigned char *begin; @@ -227,7 +228,7 @@ _parser->cur = begin; _parser->end = _parser->cur + size; - if ((ret = http_request_parse(parser, error_r)) < 0) + if ((ret = http_request_parse(parser, pool, error_r)) < 0) return -1; i_stream_skip(_parser->input, _parser->cur - begin); @@ -248,7 +249,7 @@ } int http_request_parse_next(struct http_request_parser *parser, - struct http_request *request, + pool_t pool, struct http_request *request, const char **error_r) { int ret; @@ -264,7 +265,7 @@ [ message-body ] */ if (parser->state != HTTP_REQUEST_PARSE_STATE_HEADER) { - if ((ret = http_request_parse_request_line(parser, error_r)) <= 0) + if ((ret = http_request_parse_request_line(parser, pool, error_r)) <= 0) return ret; } if ((ret = http_message_parse_headers(&parser->parser, error_r)) <= 0) diff -r adb4d013073d -r b172c130df9b src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:36:44 2013 +0300 @@ -23,7 +23,7 @@ void http_request_parser_deinit(struct http_request_parser **_parser); int http_request_parse_next(struct http_request_parser *parser, - struct http_request *request, + pool_t pool, struct http_request *request, const char **error_r); #endif diff -r adb4d013073d -r b172c130df9b src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/http-response-parser.c Sun Sep 15 03:36:44 2013 +0300 @@ -48,7 +48,7 @@ static void http_response_parser_restart(struct http_response_parser *parser) { - http_message_parser_restart(&parser->parser); + http_message_parser_restart(&parser->parser, NULL); parser->response_status = 0; parser->response_reason = NULL; } @@ -83,7 +83,7 @@ if (p == parser->parser.end) return 0; parser->response_reason = - p_strdup_until(parser->parser.msg_pool, parser->parser.cur, p); + p_strdup_until(parser->parser.msg.pool, parser->parser.cur, p); parser->parser.cur = p; return 1; } diff -r adb4d013073d -r b172c130df9b src/lib-http/test-http-server.c --- a/src/lib-http/test-http-server.c Sun Sep 15 03:36:18 2013 +0300 +++ b/src/lib-http/test-http-server.c Sun Sep 15 03:36:44 2013 +0300 @@ -53,7 +53,7 @@ int ret; while ((ret = http_request_parse_next - (client->parser, &request, &error)) > 0) { + (client->parser, NULL, &request, &error)) > 0) { if (client_handle_request(client, &request) < 0 || request.connection_close) { client_destroy(conn); From dovecot at dovecot.org Sun Sep 15 03:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:38:25 +0300 Subject: dovecot-2.2: lib-http: Added support for parsing HTTP word synta... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1404dbde402c changeset: 16742:1404dbde402c user: Stephan Bosch date: Sun Sep 15 03:37:59 2013 +0300 description: lib-http: Added support for parsing HTTP word syntax, which includes quoted-string. diffstat: src/lib-http/http-parser.c | 65 +++++++++++++++++++++++++++++++++++++++++++++- src/lib-http/http-parser.h | 3 ++ 2 files changed, 67 insertions(+), 1 deletions(-) diffs (94 lines): diff -r b172c130df9b -r 1404dbde402c src/lib-http/http-parser.c --- a/src/lib-http/http-parser.c Sun Sep 15 03:36:44 2013 +0300 +++ b/src/lib-http/http-parser.c Sun Sep 15 03:37:59 2013 +0300 @@ -23,8 +23,12 @@ ctext = OWS / %x21-27 / %x2A-5B / %x5D-7E / obs-text obs-text = %x80-FF OWS = *( SP / HTAB ) + VCHAR = %x21-7E - Mapping + 'text' = ( HTAB / SP / VCHAR / obs-text ) + + Character bit mappings: + (1<<0) => tchar (1<<1) => special (1<<2) => %x21 / %x2A-5B / %x5D-7E @@ -127,6 +131,65 @@ return 1; } +int http_parse_quoted_string(struct http_parser *parser, const char **str_r) +{ + string_t *str; + /* quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + qdtext = HTAB / SP / "!" / %x23-5B ; '#'-'[' + / %x5D-7E ; ']'-'~' + / obs-text + quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + obs-text = %x80-FF + */ + /* DQUOTE */ + if (parser->cur >= parser->end || parser->cur[0] != '"') + return 0; + parser->cur++; + /* *( qdtext / quoted-pair ) */ + str = t_str_new(256); + for (;;) { + const unsigned char *first; + + /* *qdtext */ + first = parser->cur; + while (parser->cur < parser->end && http_char_is_qdtext(*parser->cur)) + parser->cur++; + + if (parser->cur >= parser->end) + return -1; + + str_append_n(str, first, parser->cur - first); + + /* DQUOTE */ + if (*parser->cur == '"') { + parser->cur++; + break; + + /* "\" */ + } else if (*parser->cur == '\\') { + parser->cur++; + + if (parser->cur >= parser->end || !http_char_is_text(*parser->cur)) + return -1; + str_append_c(str, *parser->cur); + + /* ERROR */ + } else { + return -1; + } + } + *str_r = str_c(str); + return 1; +} + +int http_parse_word(struct http_parser *parser, const char **word_r) +{ + if (parser->cur >= parser->end) + return 0; + if (parser->cur[0] == '"') + return http_parse_quoted_string(parser, word_r); + return http_parse_token(parser, word_r); +} diff -r b172c130df9b -r 1404dbde402c src/lib-http/http-parser.h --- a/src/lib-http/http-parser.h Sun Sep 15 03:36:44 2013 +0300 +++ b/src/lib-http/http-parser.h Sun Sep 15 03:37:59 2013 +0300 @@ -50,4 +50,7 @@ int http_parse_token_list_next(struct http_parser *parser, const char **token_r); +int http_parse_quoted_string(struct http_parser *parser, const char **str_r); +int http_parse_word(struct http_parser *parser, const char **word_r); + #endif From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Unified http-request.h and http-response.... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dca140149d80 changeset: 16744:dca140149d80 user: Stephan Bosch date: Sun Sep 15 03:44:42 2013 +0300 description: lib-http: Unified http-request.h and http-response.h headers. Renamed struct http_response_header to struct http_header_field, encapsulated the array in struct http_header and put it all in http-header.h/c Added inline utility functions for header querying and getting response/request payload size. diffstat: src/lib-http/Makefile.am | 6 ++ src/lib-http/http-client-request.c | 4 +- src/lib-http/http-header.c | 98 +++++++++++++++++++++++++++++++++++++ src/lib-http/http-header.h | 39 ++++++++++++++ src/lib-http/http-message-parser.c | 39 +++++--------- src/lib-http/http-message-parser.h | 6 +- src/lib-http/http-request-parser.c | 2 +- src/lib-http/http-request-parser.h | 18 +------ src/lib-http/http-request.c | 28 ++++++++++ src/lib-http/http-request.h | 55 ++++++++++++++++++++ src/lib-http/http-response-parser.c | 3 +- src/lib-http/http-response.c | 31 +++++++++++ src/lib-http/http-response.h | 54 +++++++++++++++++-- 13 files changed, 327 insertions(+), 56 deletions(-) diffs (truncated from 565 to 300 lines): diff -r d59ac8efc5af -r dca140149d80 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Sun Sep 15 03:39:45 2013 +0300 +++ b/src/lib-http/Makefile.am Sun Sep 15 03:44:42 2013 +0300 @@ -10,10 +10,13 @@ http-date.c \ http-url.c \ http-parser.c \ + http-header.c \ http-header-parser.c \ http-transfer-chunked.c \ http-message-parser.c \ + http-request.c \ http-request-parser.c \ + http-response.c \ http-response-parser.c \ http-client-request.c \ http-client-connection.c \ @@ -25,9 +28,11 @@ http-date.h \ http-url.h \ http-parser.h \ + http-header.h \ http-header-parser.h \ http-transfer.h \ http-message-parser.h \ + http-request.h \ http-request-parser.h \ http-response.h \ http-response-parser.h \ @@ -84,6 +89,7 @@ test_http_response_parser_LDADD = \ http-date.lo \ http-parser.lo \ + http-header.lo \ http-header-parser.lo \ http-transfer-chunked.lo \ http-message-parser.lo \ diff -r d59ac8efc5af -r dca140149d80 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Sep 15 03:39:45 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:44:42 2013 +0300 @@ -513,9 +513,7 @@ if (callback != NULL) { struct http_response response; - memset(&response, 0, sizeof(response)); - response.status = status; - response.reason = error; + http_response_init(&response, status, error); (void)callback(&response, req->context); } } diff -r d59ac8efc5af -r dca140149d80 src/lib-http/http-header.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-header.c Sun Sep 15 03:44:42 2013 +0300 @@ -0,0 +1,98 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" + +#include "http-header.h" + +struct http_header { + ARRAY_TYPE(http_header_field) fields; + /* FIXME: ARRAY(struct http_header_field *) *btree; */ +}; + +struct http_header * +http_header_create(pool_t pool, unsigned int init_count) +{ + struct http_header *header; + + header = p_new(pool, struct http_header, 1); + p_array_init(&header->fields, pool, init_count); + + return header; +} + +const struct http_header_field * +http_header_field_add(struct http_header *header, + const char *name, const unsigned char *data, size_t size) +{ + struct http_header_field *hfield; + pool_t pool = array_get_pool(&header->fields); + void *value; + + hfield = array_append_space(&header->fields); + hfield->key = p_strdup(pool, name); + hfield->size = size; + + value = p_malloc(pool, size+1); + memcpy(value, data, size); + hfield->value = (const char *)value; + + return hfield; +} + +void http_header_field_delete(struct http_header *header, const char *name) +{ + ARRAY_TYPE(http_header_field) *hfields = &header->fields; + const struct http_header_field *hfield; + + array_foreach(hfields, hfield) { + if (http_header_field_is(hfield, name)) { + array_delete(hfields, array_foreach_idx(hfields, hfield), 1); + } + } +} + +const ARRAY_TYPE(http_header_field) * +http_header_get_fields(const struct http_header *header) +{ + return &header->fields; +} + +const struct http_header_field * +http_header_field_find(const struct http_header *header, const char *name) +{ + const struct http_header_field *hfield; + + array_foreach(&header->fields, hfield) { + if (http_header_field_is(hfield, name)) + return hfield; + } + + return NULL; +} + +const char * +http_header_field_get(const struct http_header *header, const char *name) +{ + const struct http_header_field *hfield = + http_header_field_find(header, name); + return (hfield == NULL ? NULL : hfield->value); +} + +int http_header_field_find_unique(const struct http_header *header, + const char *name, const struct http_header_field **hfield_r) +{ + const struct http_header_field *hfield, *hfield_found = NULL; + + array_foreach(&header->fields, hfield) { + if (http_header_field_is(hfield, name)) { + if (hfield_found != NULL) + return -1; + hfield_found = hfield; + } + } + + *hfield_r = hfield_found; + return (hfield_found == NULL ? 0 : 1); +} + diff -r d59ac8efc5af -r dca140149d80 src/lib-http/http-header.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-header.h Sun Sep 15 03:44:42 2013 +0300 @@ -0,0 +1,39 @@ +#ifndef HTTP_HEADER_H +#define HTTP_HEADER_H + +struct http_header; + +struct http_header_field { + const char *key; /* FIXME: rename to 'name' for v2.3 */ + const char *value; + size_t size; +}; +ARRAY_DEFINE_TYPE(http_header_field, struct http_header_field); + +static inline bool http_header_field_is(const struct http_header_field *hfield, + const char *name) +{ + return (strcasecmp(hfield->key, name) == 0); +} + +struct http_header * +http_header_create(pool_t pool, unsigned int init_count); + +const struct http_header_field * +http_header_field_add(struct http_header *header, + const char *name, const unsigned char *data, size_t size); +void http_header_field_delete(struct http_header *header, const char *name); + +const ARRAY_TYPE(http_header_field) * +http_header_get_fields(const struct http_header *header) ATTR_PURE; + +const struct http_header_field * +http_header_field_find(const struct http_header *header, const char *name) + ATTR_PURE; +const char * +http_header_field_get(const struct http_header *header, const char *name) + ATTR_PURE; +int http_header_field_find_unique(const struct http_header *header, + const char *name, const struct http_header_field **hfield_r) ATTR_PURE; + +#endif diff -r d59ac8efc5af -r dca140149d80 src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:39:45 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:44:42 2013 +0300 @@ -4,6 +4,7 @@ #include "array.h" #include "istream.h" #include "http-parser.h" +#include "http-header.h" #include "http-header-parser.h" #include "http-date.h" #include "http-transfer.h" @@ -48,7 +49,7 @@ pool_ref(pool); } parser->msg.date = (time_t)-1; - p_array_init(&parser->msg.headers, parser->msg.pool, 32); + parser->msg.header = http_header_create(parser->msg.pool, 32); p_array_init(&parser->msg.connection_options, parser->msg.pool, 4); } @@ -93,19 +94,14 @@ } static int -http_message_parse_header(struct http_message_parser *parser, const char *name, - const unsigned char *data, size_t size, +http_message_parse_header(struct http_message_parser *parser, + const char *name, const unsigned char *data, size_t size, const char **error_r) { - struct http_response_header *hdr; + const struct http_header_field *hdr; struct http_parser hparser; - void *value; - hdr = array_append_space(&parser->msg.headers); - hdr->key = p_strdup(parser->msg.pool, name); - hdr->value = value = p_malloc(parser->msg.pool, size+1); - memcpy(value, data, size); - hdr->size = size; + hdr = http_header_field_add(parser->msg.header, name, data, size); /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 Section 3.2.2: @@ -284,20 +280,21 @@ int http_message_parse_headers(struct http_message_parser *parser, const char **error_r) { + const unsigned char *field_data; const char *field_name, *error; - const unsigned char *field_data; size_t field_size; int ret; /* *( header-field CRLF ) CRLF */ - while ((ret=http_header_parse_next_field - (parser->header_parser, &field_name, &field_data, &field_size, &error)) > 0) { + while ((ret=http_header_parse_next_field(parser->header_parser, + &field_name, &field_data, &field_size, &error)) > 0) { if (field_name == NULL) { /* EOH */ return 1; } - if (http_message_parse_header(parser, field_name, field_data, - field_size, error_r) < 0) + + if (http_message_parse_header(parser, + field_name, field_data, field_size, error_r) < 0) return -1; } @@ -378,17 +375,9 @@ handled as an error. A sender MUST remove the received Content- Length field prior to forwarding such a message downstream. */ - if (parser->msg.have_content_length) { - ARRAY_TYPE(http_response_header) *headers = &parser->msg.headers; - const struct http_response_header *hdr; + if (parser->msg.have_content_length) + http_header_field_delete(parser->msg.header, "Content-Length"); - array_foreach(headers, hdr) { - if (strcasecmp(hdr->key, "Content-Length") == 0) { - array_delete(headers, array_foreach_idx(headers, hdr), 1); - break; - } - } - } } else if (parser->msg.content_length > 0) { /* Got explicit message size from Content-Length: header */ parser->payload = diff -r d59ac8efc5af -r dca140149d80 src/lib-http/http-message-parser.h --- a/src/lib-http/http-message-parser.h Sun Sep 15 03:39:45 2013 +0300 +++ b/src/lib-http/http-message-parser.h Sun Sep 15 03:44:42 2013 +0300 @@ -4,15 +4,17 @@ #include "http-response.h" #include "http-transfer.h" +struct http_header; + struct http_message { pool_t pool; unsigned int version_major; From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Improved message header and body parsing ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d59ac8efc5af changeset: 16743:d59ac8efc5af user: Stephan Bosch date: Sun Sep 15 03:39:45 2013 +0300 description: lib-http: Improved message header and body parsing for better RFC compliance. Added pre-parsed transfer-encoding and connection header content (array) to parsed message struct. Fixed message body handling for when both transfer-encoding and content-length headers are missing. Now duplicates of unique important message headers yield an error. diffstat: src/lib-http/http-message-parser.c | 247 +++++++++++++++++++++++++++++++---- src/lib-http/http-message-parser.h | 7 +- src/lib-http/http-request-parser.c | 3 +- src/lib-http/http-request-parser.h | 1 + src/lib-http/http-response-parser.c | 3 +- src/lib-http/http-response.h | 1 + src/lib-http/http-transfer.h | 14 ++ 7 files changed, 241 insertions(+), 35 deletions(-) diffs (truncated from 438 to 300 lines): diff -r 1404dbde402c -r d59ac8efc5af src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:37:59 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:39:45 2013 +0300 @@ -49,6 +49,7 @@ } parser->msg.date = (time_t)-1; p_array_init(&parser->msg.headers, parser->msg.pool, 32); + p_array_init(&parser->msg.connection_options, parser->msg.pool, 4); } int http_message_parse_version(struct http_message_parser *parser) @@ -106,10 +107,24 @@ memcpy(value, data, size); hdr->size = size; + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 3.2.2: + + A sender MUST NOT generate multiple header fields with the same field + name in a message unless either the entire field value for that + header field is defined as a comma-separated list [i.e., #(values)] + or the header field is a well-known exception. + */ + switch (name[0]) { case 'C': case 'c': + /* Connection: */ if (strcasecmp(name, "Connection") == 0) { + const char **opt_idx; const char *option; + unsigned int num_tokens = 0; + + /* Multiple Connection headers are allowed and combined into one */ /* Connection = 1#connection-option connection-option = token @@ -118,24 +133,42 @@ for (;;) { if (http_parse_token_list_next(&hparser, &option) <= 0) break; - if (strcasecmp(option, "close") == 0) { + num_tokens++; + if (strcasecmp(option, "close") == 0) parser->msg.connection_close = TRUE; - break; // not interested in any other options - } + opt_idx = array_append_space(&parser->msg.connection_options); + *opt_idx = p_strdup(parser->msg.pool, option); } + + if (hparser.cur < hparser.end || num_tokens == 0) { + *error_r = "Invalid Connection header"; + return -1; + } + return 0; } + /* Content-Length: */ if (strcasecmp(name, "Content-Length") == 0) { + if (parser->msg.have_content_length) { + *error_r = "Duplicate Content-Length header"; + return -1; + } /* Content-Length = 1*DIGIT */ if (str_to_uoff(hdr->value, &parser->msg.content_length) < 0) { *error_r = "Invalid Content-Length header"; return -1; } + parser->msg.have_content_length = TRUE; return 0; } break; case 'D': case 'd': if (strcasecmp(name, "Date") == 0) { + if (parser->msg.date != (time_t)-1) { + *error_r = "Duplicate Date header"; + return -1; + } + /* Date = HTTP-date */ (void)http_date_parse(data, size, &parser->msg.date); return 0; @@ -143,15 +176,102 @@ break; case 'L': case 'l': if (strcasecmp(name, "Location") == 0) { + /* FIXME: move this to response parser */ /* Location = URI-reference (not parsed here) */ parser->msg.location = hdr->value; return 0; } break; case 'T': case 't': + /* Transfer-Encoding: */ if (strcasecmp(name, "Transfer-Encoding") == 0) { - /* Transfer-Encoding = 1#transfer-coding */ - parser->msg.transfer_encoding = hdr->value; + const char *trenc = NULL; + + /* Multiple Transfer-Encoding headers are allowed and combined into one */ + if (!array_is_created(&parser->msg.transfer_encoding)) + p_array_init(&parser->msg.transfer_encoding, parser->msg.pool, 4); + + /* Transfer-Encoding = 1#transfer-coding + transfer-coding = "chunked" / "compress" / "deflate" / "gzip" + / transfer-extension + transfer-extension = token *( OWS ";" OWS transfer-parameter ) + transfer-parameter = attribute BWS "=" BWS value + attribute = token + value = word + */ + http_parser_init(&hparser, data, size); + for (;;) { + /* transfer-coding */ + if (http_parse_token(&hparser, &trenc) > 0) { + struct http_transfer_coding *coding; + bool parse_error; + + coding = array_append_space(&parser->msg.transfer_encoding); + coding->name = p_strdup(parser->msg.pool, trenc); + + /* *( OWS ";" OWS transfer-parameter ) */ + parse_error = FALSE; + for (;;) { + struct http_transfer_param *param; + const char *attribute, *value; + + /* OWS ";" OWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ';') + break; + hparser.cur++; + http_parse_ows(&hparser); + + /* attribute */ + if (http_parse_token(&hparser, &attribute) <= 0) { + parse_error = TRUE; + break; + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != '=') { + parse_error = TRUE; + break; + } + hparser.cur++; + http_parse_ows(&hparser); + + /* value */ + if (http_parse_word(&hparser, &value) <= 0) { + parse_error = TRUE; + break; + } + + if (!array_is_created(&coding->parameters)) + p_array_init(&coding->parameters, parser->msg.pool, 2); + param = array_append_space(&coding->parameters); + param->attribute = p_strdup(parser->msg.pool, attribute); + param->value = p_strdup(parser->msg.pool, value); + } + if (parse_error) + break; + + } else { + /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Appendix B: + + For compatibility with legacy list rules, recipients SHOULD accept + empty list elements. + */ + } + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ',') + break; + hparser.cur++; + http_parse_ows(&hparser); + } + + if (hparser.cur < hparser.end || + array_count(&parser->msg.transfer_encoding) == 0) { + *error_r = "Invalid Transfer-Encoding header"; + return -1; + } return 0; } break; @@ -188,40 +308,105 @@ return ret; } -int http_message_parse_body(struct http_message_parser *parser, + +int http_message_parse_body(struct http_message_parser *parser, bool request, const char **error_r) { - struct http_parser hparser; + *error_r = NULL; - if (parser->msg.content_length > 0) { + if (array_is_created(&parser->msg.transfer_encoding)) { + const struct http_transfer_coding *coding; + + bool chunked_last = FALSE; + + array_foreach(&parser->msg.transfer_encoding, coding) { + if (strcasecmp(coding->name, "chunked") == 0) { + chunked_last = TRUE; + + if (*error_r == NULL && array_is_created(&coding->parameters) && + array_count(&coding->parameters) > 0) { + const struct http_transfer_param *param = + array_idx(&coding->parameters, 0); + + *error_r = t_strdup_printf("Unexpected parameter `%s' specified" + "for the `%s' transfer coding", param->attribute, coding->name); + } + } else if (chunked_last) { + *error_r = "Chunked Transfer-Encoding must be last"; + return -1; + } else if (*error_r == NULL) { + *error_r = t_strdup_printf( + "Unknown transfer coding `%s'", coding->name); + } + } + + if (chunked_last) { + parser->payload = + http_transfer_chunked_istream_create(parser->input); + } else if (!request) { + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 3.3.3.: + + If a Transfer-Encoding header field is present in a response and + the chunked transfer coding is not the final encoding, the + message body length is determined by reading the connection until + it is closed by the server. + */ + parser->payload = + i_stream_create_limit(parser->input, (size_t)-1); + } else { + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 3.3.3.: + + If a Transfer-Encoding header field is present in a request and the + chunked transfer coding is not the final encoding, the message body + length cannot be determined reliably; the server MUST respond with + the 400 (Bad Request) status code and then close the connection. + */ + *error_r = "Final Transfer-Encoding in request is not `chunked'"; + return -1; + } + + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 3.3.3.: + + If a message is received with both a Transfer-Encoding and a + Content-Length header field, the Transfer-Encoding overrides the + Content-Length. Such a message might indicate an attempt to + perform request or response smuggling (bypass of security-related + checks on message routing or content) and thus ought to be + handled as an error. A sender MUST remove the received Content- + Length field prior to forwarding such a message downstream. + */ + if (parser->msg.have_content_length) { + ARRAY_TYPE(http_response_header) *headers = &parser->msg.headers; + const struct http_response_header *hdr; + + array_foreach(headers, hdr) { + if (strcasecmp(hdr->key, "Content-Length") == 0) { + array_delete(headers, array_foreach_idx(headers, hdr), 1); + break; + } + } + } + } else if (parser->msg.content_length > 0) { /* Got explicit message size from Content-Length: header */ parser->payload = i_stream_create_limit(parser->input, parser->msg.content_length); - } else if (parser->msg.transfer_encoding != NULL) { - const char *tenc; + } else if (!parser->msg.have_content_length && !request) { + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 3.3.3.: - /* Transfer-Encoding = 1#transfer-coding - transfer-coding = "chunked" / "compress" / "deflate" / "gzip" - / transfer-extension ; [FIXME] - transfer-extension = token *( OWS ";" OWS transfer-parameter ) - */ - http_parser_init(&hparser, - (const unsigned char *)parser->msg.transfer_encoding, - strlen(parser->msg.transfer_encoding)); - for (;;) { - if (http_parse_token_list_next(&hparser, &tenc) <= 0) - break; - if (strcasecmp(tenc, "chunked") == 0) { - parser->payload = - http_transfer_chunked_istream_create(parser->input); From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Implemented limits on overall HTTP header... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/572b9a9031e7 changeset: 16745:572b9a9031e7 user: Stephan Bosch date: Sun Sep 15 03:46:12 2013 +0300 description: lib-http: Implemented limits on overall HTTP header size, size of individual header fields and the number of fields in the header. diffstat: src/lib-http/http-client-connection.c | 3 +- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 3 + src/lib-http/http-header-parser.c | 61 ++++++++++++- src/lib-http/http-header-parser.h | 5 +- src/lib-http/http-header.h | 6 + src/lib-http/http-message-parser.c | 12 +- src/lib-http/http-message-parser.h | 6 +- src/lib-http/http-request-parser.c | 6 +- src/lib-http/http-request-parser.h | 3 +- src/lib-http/http-response-parser.c | 7 +- src/lib-http/http-response-parser.h | 4 +- src/lib-http/http-transfer-chunked.c | 4 +- src/lib-http/test-http-header-parser.c | 134 +++++++++++++++++++++++------- src/lib-http/test-http-response-parser.c | 6 +- src/lib-http/test-http-server.c | 2 +- 16 files changed, 207 insertions(+), 56 deletions(-) diffs (truncated from 605 to 300 lines): diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sun Sep 15 03:46:12 2013 +0300 @@ -721,7 +721,8 @@ } /* start protocol I/O */ - conn->http_parser = http_response_parser_init(conn->conn.input); + conn->http_parser = http_response_parser_init + (conn->conn.input, &conn->client->set.response_hdr_limits); o_stream_set_flush_callback(conn->conn.output, http_client_connection_output, conn); } diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-client.c Sun Sep 15 03:46:12 2013 +0300 @@ -96,6 +96,7 @@ (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); client->set.max_attempts = set->max_attempts; client->set.max_redirects = set->max_redirects; + client->set.response_hdr_limits = set->response_hdr_limits; client->set.request_timeout_msecs = set->request_timeout_msecs; client->set.connect_timeout_msecs = set->connect_timeout_msecs; client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs; diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-client.h Sun Sep 15 03:46:12 2013 +0300 @@ -59,6 +59,9 @@ /* maximum number of attempts for a request */ unsigned int max_attempts; + /* response header limits */ + struct http_header_limits response_hdr_limits; + /* max time to wait for HTTP request to finish before retrying (default = unlimited) */ unsigned int request_timeout_msecs; diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-header-parser.c --- a/src/lib-http/http-header-parser.c Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-header-parser.c Sun Sep 15 03:46:12 2013 +0300 @@ -6,6 +6,7 @@ #include "str.h" #include "str-sanitize.h" #include "http-parser.h" +#include "http-header.h" #include "http-header-parser.h" @@ -25,6 +26,10 @@ struct http_header_parser { struct istream *input; + struct http_header_limits limits; + uoff_t size, field_size; + unsigned int field_count; + const unsigned char *begin, *cur, *end; const char *error; @@ -34,9 +39,9 @@ enum http_header_parse_state state; }; -// FIXME(Stephan): Add support for limiting maximum header size. - -struct http_header_parser *http_header_parser_init(struct istream *input) +struct http_header_parser * +http_header_parser_init(struct istream *input, + const struct http_header_limits *limits) { struct http_header_parser *parser; @@ -45,6 +50,16 @@ parser->name = str_new(default_pool, 128); parser->value_buf = buffer_create_dynamic(default_pool, 4096); + if (limits != NULL) + parser->limits = *limits; + + if (parser->limits.max_size == 0) + parser->limits.max_size = (uoff_t)-1; + if (parser->limits.max_field_size == 0) + parser->limits.max_field_size = (uoff_t)-1; + if (parser->limits.max_fields == 0) + parser->limits.max_fields = (unsigned int)-1; + return parser; } @@ -62,6 +77,9 @@ void http_header_parser_reset(struct http_header_parser *parser) { parser->state = HTTP_HEADER_PARSE_STATE_INIT; + parser->size = 0; + parser->field_size = 0; + parser->field_count = 0; } static int http_header_parse_name(struct http_header_parser *parser) @@ -144,7 +162,7 @@ if (http_char_is_token(*parser->cur)) { if ((ret=http_header_parse_name(parser)) <= 0) return ret; - } else if (str_len(parser->name) == 0) { + } else if (*parser->cur != ':' && str_len(parser->name) == 0) { parser->state = HTTP_HEADER_PARSE_STATE_LAST_LINE; break; } @@ -163,6 +181,10 @@ parser->error = "Empty header field name"; return -1; } + if (++parser->field_count > parser->limits.max_fields) { + parser->error = "Excessive number of header fields"; + return -1; + } parser->state = HTTP_HEADER_PARSE_STATE_OWS; /* fall through */ case HTTP_HEADER_PARSE_STATE_OWS: @@ -203,7 +225,7 @@ parser->state = HTTP_HEADER_PARSE_STATE_OWS; break; } - parser->state = HTTP_HEADER_PARSE_STATE_NAME; + parser->state = HTTP_HEADER_PARSE_STATE_INIT; return 1; case HTTP_HEADER_PARSE_STATE_LAST_LINE: if (*parser->cur == '\r') { @@ -247,12 +269,35 @@ const char **name_r, const unsigned char **data_r, size_t *size_r, const char **error_r) { + const uoff_t max_size = parser->limits.max_size; + const uoff_t max_field_size = parser->limits.max_field_size; const unsigned char *data; - size_t size; + uoff_t size; int ret; + *error_r = NULL; + while ((ret=i_stream_read_data (parser->input, &parser->begin, &size, 0)) > 0) { + + /* check header size limits */ + if (parser->size >= max_size) { + *error_r = "Excessive header size"; + return -1; + } + if (parser->field_size > max_field_size) { + *error_r = "Excessive header field size"; + return -1; + } + + /* don't parse beyond header size limits */ + if (size > (max_size - parser->size)) + size = max_size - parser->size; + if (size > (max_field_size - parser->field_size)) { + size = max_field_size - parser->field_size; + size = (size == 0 ? 1 : size); /* need to parse one more byte */ + } + parser->cur = parser->begin; parser->end = parser->cur + size; @@ -262,8 +307,12 @@ } i_stream_skip(parser->input, parser->cur - parser->begin); + parser->size += parser->cur - parser->begin; + parser->field_size += parser->cur - parser->begin; if (ret == 1) { + parser->field_size = 0; + if (parser->state != HTTP_HEADER_PARSE_STATE_EOH) { data = buffer_get_data(parser->value_buf, &size); diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-header-parser.h --- a/src/lib-http/http-header-parser.h Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-header-parser.h Sun Sep 15 03:46:12 2013 +0300 @@ -1,9 +1,12 @@ #ifndef HTTP_HEADER_PARSER_H #define HTTP_HEADER_PARSER_H +struct http_header_limits; struct http_header_parser; -struct http_header_parser *http_header_parser_init(struct istream *input); +struct http_header_parser * +http_header_parser_init(struct istream *input, + const struct http_header_limits *limits); void http_header_parser_deinit(struct http_header_parser **_parser); void http_header_parser_reset(struct http_header_parser *parser); diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-header.h --- a/src/lib-http/http-header.h Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-header.h Sun Sep 15 03:46:12 2013 +0300 @@ -3,6 +3,12 @@ struct http_header; +struct http_header_limits { + uoff_t max_size; + uoff_t max_field_size; + unsigned int max_fields; +}; + struct http_header_field { const char *key; /* FIXME: rename to 'name' for v2.3 */ const char *value; diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:46:12 2013 +0300 @@ -13,10 +13,12 @@ #include void http_message_parser_init(struct http_message_parser *parser, - struct istream *input) + struct istream *input, const struct http_header_limits *hdr_limits) { memset(parser, 0, sizeof(*parser)); parser->input = input; + if (hdr_limits != NULL) + parser->header_limits = *hdr_limits; } void http_message_parser_deinit(struct http_message_parser *parser) @@ -34,10 +36,12 @@ { i_assert(parser->payload == NULL); - if (parser->header_parser == NULL) - parser->header_parser = http_header_parser_init(parser->input); - else + if (parser->header_parser == NULL) { + parser->header_parser = + http_header_parser_init(parser->input, &parser->header_limits); + } else { http_header_parser_reset(parser->header_parser); + } if (parser->msg.pool != NULL) pool_unref(&parser->msg.pool); diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-message-parser.h --- a/src/lib-http/http-message-parser.h Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-message-parser.h Sun Sep 15 03:46:12 2013 +0300 @@ -4,7 +4,7 @@ #include "http-response.h" #include "http-transfer.h" -struct http_header; +#include "http-header.h" struct http_message { pool_t pool; @@ -26,6 +26,7 @@ struct http_message_parser { struct istream *input; + struct http_header_limits header_limits; const unsigned char *cur, *end; @@ -37,7 +38,8 @@ }; void http_message_parser_init(struct http_message_parser *parser, - struct istream *input); + struct istream *input, const struct http_header_limits *hdr_limits) + ATTR_NULL(3); void http_message_parser_deinit(struct http_message_parser *parser); void http_message_parser_restart(struct http_message_parser *parser, pool_t pool); diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:46:12 2013 +0300 @@ -29,12 +29,14 @@ unsigned int skipping_line:1; }; -struct http_request_parser *http_request_parser_init(struct istream *input) +struct http_request_parser * +http_request_parser_init(struct istream *input, + const struct http_header_limits *hdr_limits) { struct http_request_parser *parser; parser = i_new(struct http_request_parser, 1); - http_message_parser_init(&parser->parser, input); + http_message_parser_init(&parser->parser, input, hdr_limits); return parser; } diff -r dca140149d80 -r 572b9a9031e7 src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:44:42 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:46:12 2013 +0300 @@ -4,7 +4,8 @@ #include "http-request.h" From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Improved display of invalid characters in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bbe4a469e276 changeset: 16746:bbe4a469e276 user: Stephan Bosch date: Sun Sep 15 03:46:25 2013 +0300 description: lib-http: Improved display of invalid characters in parse error messages. diffstat: src/lib-http/http-request-parser.c | 8 ++++++-- src/lib-http/http-response-parser.c | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diffs (36 lines): diff -r 572b9a9031e7 -r bbe4a469e276 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:46:12 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:46:25 2013 +0300 @@ -96,8 +96,12 @@ static inline const char *_chr_sanitize(unsigned char c) { if (c >= 0x20 && c < 0x7F) - return t_strdup_printf("'%c'", c); - return t_strdup_printf("0x%02x", c); + return t_strdup_printf("`%c'", c); + if (c == 0x0a) + return ""; + if (c == 0x0d) + return ""; + return t_strdup_printf("<0x%02x>", c); } static int http_request_parse(struct http_request_parser *parser, diff -r 572b9a9031e7 -r bbe4a469e276 src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Sun Sep 15 03:46:12 2013 +0300 +++ b/src/lib-http/http-response-parser.c Sun Sep 15 03:46:25 2013 +0300 @@ -94,8 +94,12 @@ static inline const char *_chr_sanitize(unsigned char c) { if (c >= 0x20 && c < 0x7F) - return t_strdup_printf("'%c'", c); - return t_strdup_printf("0x%02x", c); + return t_strdup_printf("`%c'", c); + if (c == 0x0a) + return ""; + if (c == 0x0d) + return ""; + return t_strdup_printf("<0x%02x>", c); } static int http_response_parse(struct http_response_parser *parser, From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Added support for parsing request target ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/29ceb7126b91 changeset: 16747:29ceb7126b91 user: Stephan Bosch date: Sun Sep 15 03:47:29 2013 +0300 description: lib-http: Added support for parsing request target URLs. diffstat: src/lib-http/http-request.h | 14 ++ src/lib-http/http-url.c | 286 ++++++++++++++++++++++++++++++++++--------- src/lib-http/http-url.h | 6 + 3 files changed, 245 insertions(+), 61 deletions(-) diffs (truncated from 435 to 300 lines): diff -r bbe4a469e276 -r 29ceb7126b91 src/lib-http/http-request.h --- a/src/lib-http/http-request.h Sun Sep 15 03:46:25 2013 +0300 +++ b/src/lib-http/http-request.h Sun Sep 15 03:47:29 2013 +0300 @@ -3,6 +3,20 @@ #include "http-header.h" +struct http_url; + +enum http_request_target_format { + HTTP_REQUEST_TARGET_FORMAT_ORIGIN = 0, + HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + HTTP_REQUEST_TARGET_FORMAT_AUTHORITY, + HTTP_REQUEST_TARGET_FORMAT_ASTERISK +}; + +struct http_request_target { + enum http_request_target_format format; + struct http_url *url; +}; + struct http_request { const char *method; diff -r bbe4a469e276 -r 29ceb7126b91 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Sun Sep 15 03:46:25 2013 +0300 +++ b/src/lib-http/http-url.c Sun Sep 15 03:47:29 2013 +0300 @@ -5,7 +5,9 @@ #include "strfuncs.h" #include "net.h" #include "uri-util.h" + #include "http-url.h" +#include "http-request.h" /* * HTTP URL parser @@ -19,57 +21,20 @@ struct http_url *url; struct http_url *base; - unsigned int relative:1; + enum http_request_target_format req_format; + + unsigned int relative:1; + unsigned int request_target:1; }; -static bool http_url_do_parse(struct http_url_parser *url_parser) +static bool http_url_parse_authority(struct http_url_parser *url_parser) { struct uri_parser *parser = &url_parser->parser; - struct http_url *url = url_parser->url, *base = url_parser->base; + struct http_url *url = url_parser->url; struct uri_authority auth; - const char *const *path; - bool relative = TRUE, have_path = FALSE; - int path_relative; - const char *part; int ret; - /* RFC 2616 - Hypertext Transfer Protocol, Section 3.2: - * - * http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] - * - * Translated to RFC 3986: - * - * absolute-http-URL = "http:" "//" host [ ":" port ] path-absolute - * ["?" query] [ "#" fragment ] - * relative-http-ref = relative-http-part [ "?" query ] [ "#" fragment ] - * relative-http-part = "//" host [ ":" port ] path-abempty - * / path-absolute - * / path-noscheme - * / path-empty - */ - - /* "http:" / "https:" */ - if ((url_parser->flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0) { - const char *scheme; - - if ((ret = uri_parse_scheme(parser, &scheme)) < 0) - return FALSE; - else if (ret > 0) { - if (strcasecmp(scheme, "https") == 0) { - if (url != NULL) - url->have_ssl = TRUE; - } else if (strcasecmp(scheme, "http") != 0) { - parser->error = "Not an HTTP URL"; - return FALSE; - } - relative = FALSE; - } - } else { - relative = FALSE; - } - - /* "//" host [ ":" port ] */ - if ((ret = uri_parse_slashslash_authority(parser, &auth)) < 0) + if ((ret = uri_parse_authority(parser, &auth)) < 0) return FALSE; if (ret > 0) { if (auth.enc_userinfo != NULL) { @@ -87,19 +52,153 @@ parser->error = "HTTP URL does not allow `userinfo@' part"; return FALSE; } - relative = FALSE; - } else if (!relative) { - parser->error = "Absolute HTTP URL requires `//' after `http:'"; - return FALSE; } - - if (ret > 0 && url != NULL) { + if (url != NULL) { url->host_name = p_strdup(parser->pool, auth.host_literal); url->host_ip = auth.host_ip; url->have_host_ip = auth.have_host_ip; url->port = auth.port; url->have_port = auth.have_port; } + return TRUE; +} + +static bool http_url_parse_authority_form(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + + if (!http_url_parse_authority(url_parser)) + return FALSE; + if (parser->cur != parser->end) + return FALSE; + url_parser->req_format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY; + return TRUE; +} + +static bool http_url_do_parse(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + const char *const *path; + bool relative = TRUE, have_scheme = FALSE, have_authority = FALSE, + have_path = FALSE; + int path_relative; + const char *part; + int ret; + + /* + http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Appendix C: + + http-URI = "http://" authority path-abempty [ "?" query ] + [ "#" fragment ] + https-URI = "https://" authority path-abempty [ "?" query ] + [ "#" fragment ] + partial-URI = relative-part [ "?" query ] + + request-target = origin-form / absolute-form / authority-form / + asterisk-form + + origin-form = absolute-path [ "?" query ] + absolute-form = absolute-URI + authority-form = authority + asterisk-form = "*" + ; Not parsed here + + absolute-path = 1*( "/" segment ) + + http://tools.ietf.org/html/rfc3986 + Appendix A: (implemented in uri-util.h) + + absolute-URI = scheme ":" hier-part [ "?" query ] + + hier-part = "//" authority path-abempty + / path-absolute + / path-rootless + / path-empty + + relative-part = "//" authority path-abempty + / path-absolute + / path-noscheme + / path-empty + + authority = [ userinfo "@" ] host [ ":" port ] + + path-abempty = *( "/" segment ) + path-absolute = "/" [ segment-nz *( "/" segment ) ] + path-noscheme = segment-nz-nc *( "/" segment ) + path-rootless = segment-nz *( "/" segment ) + path-empty = 0 + + segment = *pchar + segment-nz = 1*pchar + segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + ; non-zero-length segment without any colon ":" + + query = *( pchar / "/" / "?" ) + fragment = *( pchar / "/" / "?" ) + */ + + /* "http:" / "https:" */ + if ((url_parser->flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0) { + const char *scheme; + + if ((ret = uri_parse_scheme(parser, &scheme)) < 0) + return FALSE; + else if (ret > 0) { + if (strcasecmp(scheme, "https") == 0) { + if (url != NULL) + url->have_ssl = TRUE; + } else if (strcasecmp(scheme, "http") != 0) { + if (url_parser->request_target) { + /* valid as non-HTTP scheme, but also try to parse as authority */ + parser->cur = parser->begin; + if (!http_url_parse_authority_form(url_parser)) { + url_parser->url = NULL; /* indicate non-http-url */ + url_parser->req_format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE; + } + return TRUE; + } + parser->error = "Not an HTTP URL"; + return FALSE; + } + relative = FALSE; + have_scheme = TRUE; + } + } else { + relative = FALSE; + have_scheme = TRUE; + } + + /* "//" authority ; or + * ["//"] authority ; when parsing a request target + */ + if (parser->cur < parser->end && parser->cur[0] == '/') { + if (parser->cur+1 < parser->end && parser->cur[1] == '/') { + parser->cur += 2; + relative = FALSE; + have_authority = TRUE; + } else { + /* start of absolute-path */ + } + } else if (url_parser->request_target && !have_scheme) { + if (!http_url_parse_authority_form(url_parser)) { + /* not non-HTTP scheme and invalid as authority-form */ + parser->error = "Request target is invalid"; + return FALSE; + } + return TRUE; + } + + if (have_scheme && !have_authority) { + parser->error = "Absolute HTTP URL requires `//' after `http:'"; + return FALSE; + } + + if (have_authority) { + if (!http_url_parse_authority(url_parser)) + return FALSE; + } /* path-abempty / path-absolute / path-noscheme / path-empty */ if ((ret = uri_parse_path(parser, &path_relative, &path)) < 0) @@ -108,14 +207,15 @@ /* Relative URLs are only valid when we have a base URL */ if (relative) { if (base == NULL) { - parser->error = "Relative URL not allowed"; + parser->error = "Relative HTTP URL not allowed"; return FALSE; - } else if (url != NULL) { - url->host_name = p_strdup_empty(parser->pool, base->host_name); + } else if (!have_authority && url != NULL) { + url->host_name = p_strdup(parser->pool, base->host_name); url->host_ip = base->host_ip; url->have_host_ip = base->have_host_ip; url->port = base->port; url->have_port = base->have_port; + url->have_ssl = base->have_ssl; } url_parser->relative = TRUE; @@ -152,7 +252,7 @@ if (url != NULL && pend > pbegin) str_append_n(fullpath, pbegin, pend-pbegin); } - + /* append relative path */ while (*path != NULL) { if (!uri_data_decode(parser, *path, NULL, &part)) @@ -161,7 +261,7 @@ if (url != NULL) { str_append_c(fullpath, '/'); str_append(fullpath, part); - } + } path++; } @@ -170,7 +270,7 @@ From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Adjusted request parser to pre-parse the ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eeaa68773f73 changeset: 16748:eeaa68773f73 user: Stephan Bosch date: Sun Sep 15 03:47:54 2013 +0300 description: lib-http: Adjusted request parser to pre-parse the request target and host header into a proper target url. diffstat: src/lib-http/http-request-parser.c | 30 +++++++++++++++++++++++++++++- src/lib-http/http-request.h | 3 ++- src/lib-http/test-http-server.c | 4 ++-- 3 files changed, 33 insertions(+), 4 deletions(-) diffs (85 lines): diff -r 29ceb7126b91 -r eeaa68773f73 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:47:29 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:47:54 2013 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "istream.h" +#include "http-url.h" #include "http-parser.h" #include "http-message-parser.h" #include "http-request-parser.h" @@ -258,6 +259,8 @@ pool_t pool, struct http_request *request, const char **error_r) { + const struct http_header_field *hdr; + const char *error; int ret; /* make sure we finished streaming payload from previous request @@ -280,9 +283,34 @@ return -1; parser->state = HTTP_REQUEST_PARSE_STATE_INIT; + /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 + Section 5.4: + + A server MUST respond with a 400 (Bad Request) status code to any + HTTP/1.1 request message that lacks a Host header field and to any + request message that contains more than one Host header field or a + Host header field with an invalid field-value. + */ + if ((ret=http_header_field_find_unique + (parser->parser.msg.header, "Host", &hdr)) <= 0) { + if (ret == 0) + *error_r = "Missing Host header"; + else + *error_r = "Duplicate Host header"; + return -1; + } + memset(request, 0, sizeof(*request)); + + if (http_url_request_target_parse(parser->request_target, hdr->value, + parser->parser.msg.pool, &request->target, &error) < 0) { + *error_r = t_strdup_printf("Bad request target `%s': %s", + parser->request_target, error); + return -1; + } + request->method = parser->request_method; - request->target = parser->request_target; + request->target_raw = parser->request_target; request->version_major = parser->parser.msg.version_major; request->version_minor = parser->parser.msg.version_minor; request->date = parser->parser.msg.date; diff -r 29ceb7126b91 -r eeaa68773f73 src/lib-http/http-request.h --- a/src/lib-http/http-request.h Sun Sep 15 03:47:29 2013 +0300 +++ b/src/lib-http/http-request.h Sun Sep 15 03:47:54 2013 +0300 @@ -20,7 +20,8 @@ struct http_request { const char *method; - const char *target; + const char *target_raw; + struct http_request_target target; unsigned char version_major; unsigned char version_minor; diff -r 29ceb7126b91 -r eeaa68773f73 src/lib-http/test-http-server.c --- a/src/lib-http/test-http-server.c Sun Sep 15 03:47:29 2013 +0300 +++ b/src/lib-http/test-http-server.c Sun Sep 15 03:47:54 2013 +0300 @@ -37,10 +37,10 @@ } str_append(str, "HTTP/1.1 200 OK\r\n"); str_printfa(str, "Date: %s\r\n", http_date_create(ioloop_time)); - str_printfa(str, "Content-Length: %d\r\n", (int)strlen(request->target)); + str_printfa(str, "Content-Length: %d\r\n", (int)strlen(request->target_raw)); str_append(str, "Content-Type: text/plain\r\n"); str_append(str, "\r\n"); - str_append(str, request->target); + str_append(str, request->target_raw); o_stream_send(client->conn.output, str_data(str), str_len(str)); return 0; } From dovecot at dovecot.org Sun Sep 15 03:50:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:50:34 +0300 Subject: dovecot-2.2: lib-http: Adjusted message and request parsers to r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/28e5d58e49dc changeset: 16749:28e5d58e49dc user: Stephan Bosch date: Sun Sep 15 03:50:08 2013 +0300 description: lib-http: Adjusted message and request parsers to return an error code. diffstat: src/lib-http/http-message-parser.c | 91 +++++++++++----- src/lib-http/http-message-parser.h | 21 ++- src/lib-http/http-request-parser.c | 117 ++++++++++++++++----- src/lib-http/http-request-parser.h | 13 ++- src/lib-http/http-response-parser.c | 189 ++++++++++++++++++----------------- src/lib-http/test-http-server.c | 3 +- 6 files changed, 277 insertions(+), 157 deletions(-) diffs (truncated from 815 to 300 lines): diff -r eeaa68773f73 -r 28e5d58e49dc src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:47:54 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:50:08 2013 +0300 @@ -62,35 +62,45 @@ const unsigned char *p = parser->cur; const size_t size = parser->end - parser->cur; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + /* HTTP-version = HTTP-name "/" DIGIT "." DIGIT HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive */ if (size < 8) return 0; if (memcmp(p, "HTTP/", 5) != 0 || - !i_isdigit(p[5]) || p[6] != '.' || !i_isdigit(p[7])) + !i_isdigit(p[5]) || p[6] != '.' || !i_isdigit(p[7])) { + parser->error = "Bad HTTP version"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; + } parser->msg.version_major = p[5] - '0'; parser->msg.version_minor = p[7] - '0'; parser->cur += 8; return 1; } -int http_message_parse_finish_payload(struct http_message_parser *parser, - const char **error_r) +int http_message_parse_finish_payload(struct http_message_parser *parser) { const unsigned char *data; size_t size; int ret; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + if (parser->payload == NULL) return 1; while ((ret = i_stream_read_data(parser->payload, &data, &size, 0)) > 0) i_stream_skip(parser->payload, size); if (ret == 0 || parser->payload->stream_errno != 0) { - if (ret < 0) - *error_r = "Stream error while skipping payload"; + if (ret < 0) { + parser->error = "Stream error while skipping payload"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + } return ret; } i_stream_unref(&parser->payload); @@ -99,8 +109,7 @@ static int http_message_parse_header(struct http_message_parser *parser, - const char *name, const unsigned char *data, size_t size, - const char **error_r) + const char *name, const unsigned char *data, size_t size) { const struct http_header_field *hdr; struct http_parser hparser; @@ -141,7 +150,8 @@ } if (hparser.cur < hparser.end || num_tokens == 0) { - *error_r = "Invalid Connection header"; + parser->error = "Invalid Connection header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; } @@ -150,12 +160,14 @@ /* Content-Length: */ if (strcasecmp(name, "Content-Length") == 0) { if (parser->msg.have_content_length) { - *error_r = "Duplicate Content-Length header"; + parser->error = "Duplicate Content-Length header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; } /* Content-Length = 1*DIGIT */ if (str_to_uoff(hdr->value, &parser->msg.content_length) < 0) { - *error_r = "Invalid Content-Length header"; + parser->error= "Invalid Content-Length header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; } parser->msg.have_content_length = TRUE; @@ -165,7 +177,8 @@ case 'D': case 'd': if (strcasecmp(name, "Date") == 0) { if (parser->msg.date != (time_t)-1) { - *error_r = "Duplicate Date header"; + parser->error = "Duplicate Date header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; } @@ -269,7 +282,8 @@ if (hparser.cur < hparser.end || array_count(&parser->msg.transfer_encoding) == 0) { - *error_r = "Invalid Transfer-Encoding header"; + parser->error = "Invalid Transfer-Encoding header"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; return -1; } return 0; @@ -281,14 +295,16 @@ return 0; } -int http_message_parse_headers(struct http_message_parser *parser, - const char **error_r) +int http_message_parse_headers(struct http_message_parser *parser) { const unsigned char *field_data; const char *field_name, *error; size_t field_size; int ret; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + /* *( header-field CRLF ) CRLF */ while ((ret=http_header_parse_next_field(parser->header_parser, &field_name, &field_data, &field_size, &error)) > 0) { @@ -298,23 +314,28 @@ } if (http_message_parse_header(parser, - field_name, field_data, field_size, error_r) < 0) + field_name, field_data, field_size) < 0) return -1; } if (ret < 0) { - *error_r = t_strdup_printf( - "Failed to parse response header: %s", error); + if (parser->input->eof || parser->input->stream_errno != 0) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + parser->error = "Broken stream"; + } else { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = t_strdup_printf("Failed to parse header: %s", error); + } + } return ret; } - -int http_message_parse_body(struct http_message_parser *parser, bool request, - const char **error_r) +int http_message_parse_body(struct http_message_parser *parser, bool request) { - *error_r = NULL; - + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NONE; + parser->error = NULL; + if (array_is_created(&parser->msg.transfer_encoding)) { const struct http_transfer_coding *coding; @@ -324,20 +345,27 @@ if (strcasecmp(coding->name, "chunked") == 0) { chunked_last = TRUE; - if (*error_r == NULL && array_is_created(&coding->parameters) && - array_count(&coding->parameters) > 0) { + if ((parser->error_code == HTTP_MESSAGE_PARSE_ERROR_NONE) + && array_is_created(&coding->parameters) + && array_count(&coding->parameters) > 0) { const struct http_transfer_param *param = array_idx(&coding->parameters, 0); - - *error_r = t_strdup_printf("Unexpected parameter `%s' specified" + + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE; + parser->error = t_strdup_printf( + "Unexpected parameter `%s' specified" "for the `%s' transfer coding", param->attribute, coding->name); + /* recoverable */ } } else if (chunked_last) { - *error_r = "Chunked Transfer-Encoding must be last"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = "Chunked Transfer-Encoding must be last"; return -1; - } else if (*error_r == NULL) { - *error_r = t_strdup_printf( + } else if (parser->error_code == HTTP_MESSAGE_PARSE_ERROR_NONE) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED; + parser->error = t_strdup_printf( "Unknown transfer coding `%s'", coding->name); + /* recoverable */ } } @@ -364,7 +392,8 @@ length cannot be determined reliably; the server MUST respond with the 400 (Bad Request) status code and then close the connection. */ - *error_r = "Final Transfer-Encoding in request is not `chunked'"; + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = "Final Transfer-Encoding in request is not chunked"; return -1; } @@ -401,5 +430,7 @@ parser->payload = i_stream_create_limit(parser->input, (size_t)-1); } + if (parser->error_code != HTTP_MESSAGE_PARSE_ERROR_NONE) + return -1; return 0; } diff -r eeaa68773f73 -r 28e5d58e49dc src/lib-http/http-message-parser.h --- a/src/lib-http/http-message-parser.h Sun Sep 15 03:47:54 2013 +0300 +++ b/src/lib-http/http-message-parser.h Sun Sep 15 03:50:08 2013 +0300 @@ -6,6 +6,15 @@ #include "http-header.h" +enum http_message_parse_error { + HTTP_MESSAGE_PARSE_ERROR_NONE = 0, /* no error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM, /* stream error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE, /* unrecoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE, /* recoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature + (recoverable) */ +}; + struct http_message { pool_t pool; @@ -30,6 +39,9 @@ const unsigned char *cur, *end; + const char *error; + enum http_message_parse_error error_code; + struct http_header_parser *header_parser; struct istream *payload; @@ -44,12 +56,9 @@ void http_message_parser_restart(struct http_message_parser *parser, pool_t pool); -int http_message_parse_finish_payload(struct http_message_parser *parser, - const char **error_r); +int http_message_parse_finish_payload(struct http_message_parser *parser); int http_message_parse_version(struct http_message_parser *parser); -int http_message_parse_headers(struct http_message_parser *parser, - const char **error_r); -int http_message_parse_body(struct http_message_parser *parser, bool request, - const char **error_r); +int http_message_parse_headers(struct http_message_parser *parser); +int http_message_parse_body(struct http_message_parser *parser, bool request); #endif diff -r eeaa68773f73 -r 28e5d58e49dc src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:47:54 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:50:08 2013 +0300 @@ -24,6 +24,8 @@ struct http_message_parser parser; enum http_request_parser_state state; + enum http_request_parse_error error_code; + const char *request_method; const char *request_target; @@ -106,7 +108,7 @@ } static int http_request_parse(struct http_request_parser *parser, - pool_t pool, const char **error_r) + pool_t pool) { struct http_message_parser *_parser = &parser->parser; int ret; @@ -125,7 +127,8 @@ if (*_parser->cur == '\r' || *_parser->cur == '\n') { if (parser->skipping_line) { /* second extra CRLF; not allowed */ - *error_r = "Empty request line"; + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; + _parser->error = "Empty request line"; return -1; } /* HTTP/1.0 client sent one extra CRLF after body. @@ -138,19 +141,17 @@ parser->skipping_line = FALSE; /* fall through */ case HTTP_REQUEST_PARSE_STATE_METHOD: From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: Added support for enforcing a payload lim... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a335db9dca6a changeset: 16750:a335db9dca6a user: Stephan Bosch date: Sun Sep 15 03:52:01 2013 +0300 description: lib-http: Added support for enforcing a payload limit for incoming HTTP messages. diffstat: src/lib-http/http-message-parser.c | 31 +++++++++++++++++++++++++------ src/lib-http/http-message-parser.h | 20 ++++++++++++-------- src/lib-http/http-request-parser.c | 2 +- src/lib-http/http-response-parser.c | 2 +- src/lib-http/http-transfer-chunked.c | 14 ++++++++++++-- src/lib-http/http-transfer.h | 2 +- src/lib-http/test-http-transfer.c | 6 +++--- 7 files changed, 55 insertions(+), 22 deletions(-) diffs (235 lines): diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:52:01 2013 +0300 @@ -13,12 +13,14 @@ #include void http_message_parser_init(struct http_message_parser *parser, - struct istream *input, const struct http_header_limits *hdr_limits) + struct istream *input, const struct http_header_limits *hdr_limits, + uoff_t max_payload_size) { memset(parser, 0, sizeof(*parser)); parser->input = input; if (hdr_limits != NULL) parser->header_limits = *hdr_limits; + parser->max_payload_size = max_payload_size; } void http_message_parser_deinit(struct http_message_parser *parser) @@ -98,8 +100,16 @@ i_stream_skip(parser->payload, size); if (ret == 0 || parser->payload->stream_errno != 0) { if (ret < 0) { - parser->error = "Stream error while skipping payload"; - parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + if (parser->payload->stream_errno == EMSGSIZE) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE; + parser->error = "Payload is too large"; + } else if (parser->payload->stream_errno == EIO) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE; + parser->error = "Invalid payload"; + } else { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM; + parser->error = "Stream error while skipping payload"; + } } return ret; } @@ -370,8 +380,8 @@ } if (chunked_last) { - parser->payload = - http_transfer_chunked_istream_create(parser->input); + parser->payload = http_transfer_chunked_istream_create + (parser->input, parser->max_payload_size); } else if (!request) { /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 Section 3.3.3.: @@ -381,7 +391,8 @@ message body length is determined by reading the connection until it is closed by the server. */ - parser->payload = + /* FIXME: enforce max payload size (relevant to http-client only) */ + parser->payload = i_stream_create_limit(parser->input, (size_t)-1); } else { /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-23 @@ -412,6 +423,13 @@ http_header_field_delete(parser->msg.header, "Content-Length"); } else if (parser->msg.content_length > 0) { + if (parser->max_payload_size > 0 + && parser->msg.content_length > parser->max_payload_size) { + parser->error_code = HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE; + parser->error = "Payload is too large"; + return -1; + } + /* Got explicit message size from Content-Length: header */ parser->payload = i_stream_create_limit(parser->input, @@ -427,6 +445,7 @@ body length, so the message body length is determined by the number of octets received prior to the server closing the connection. */ + /* FIXME: enforce max payload size (relevant to http-client only) */ parser->payload = i_stream_create_limit(parser->input, (size_t)-1); } diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-message-parser.h --- a/src/lib-http/http-message-parser.h Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-message-parser.h Sun Sep 15 03:52:01 2013 +0300 @@ -7,12 +7,14 @@ #include "http-header.h" enum http_message_parse_error { - HTTP_MESSAGE_PARSE_ERROR_NONE = 0, /* no error */ - HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM, /* stream error */ - HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE, /* unrecoverable generic error */ - HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE, /* recoverable generic error */ - HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature - (recoverable) */ + HTTP_MESSAGE_PARSE_ERROR_NONE = 0, /* no error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_STREAM, /* stream error */ + HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE, /* unrecoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_BAD_MESSAGE, /* recoverable generic error */ + HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature + (recoverable) */ + HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE /* message payload is too large + (fatal) */ }; struct http_message { @@ -35,7 +37,9 @@ struct http_message_parser { struct istream *input; + struct http_header_limits header_limits; + uoff_t max_payload_size; const unsigned char *cur, *end; @@ -50,8 +54,8 @@ }; void http_message_parser_init(struct http_message_parser *parser, - struct istream *input, const struct http_header_limits *hdr_limits) - ATTR_NULL(3); + struct istream *input, const struct http_header_limits *hdr_limits, + uoff_t max_payload_size) ATTR_NULL(3); void http_message_parser_deinit(struct http_message_parser *parser); void http_message_parser_restart(struct http_message_parser *parser, pool_t pool); diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:52:01 2013 +0300 @@ -39,7 +39,7 @@ struct http_request_parser *parser; parser = i_new(struct http_request_parser, 1); - http_message_parser_init(&parser->parser, input, hdr_limits); + http_message_parser_init(&parser->parser, input, hdr_limits, 0); return parser; } diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-response-parser.c --- a/src/lib-http/http-response-parser.c Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-response-parser.c Sun Sep 15 03:52:01 2013 +0300 @@ -36,7 +36,7 @@ /* FIXME: implement status line limit */ parser = i_new(struct http_response_parser, 1); - http_message_parser_init(&parser->parser, input, hdr_limits); + http_message_parser_init(&parser->parser, input, hdr_limits, 0); return parser; } diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-transfer-chunked.c --- a/src/lib-http/http-transfer-chunked.c Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-transfer-chunked.c Sun Sep 15 03:52:01 2013 +0300 @@ -43,6 +43,7 @@ unsigned int parsed_chars; uoff_t chunk_size, chunk_v_offset, chunk_pos; + uoff_t size, max_size; const char *error; struct http_header_parser *header_parser; @@ -327,8 +328,16 @@ i_stream_skip(input, tcstream->cur - tcstream->begin); if (ret > 0) { - if (tcstream->state == HTTP_CHUNKED_PARSE_STATE_DATA) + if (tcstream->state == HTTP_CHUNKED_PARSE_STATE_DATA) { tcstream->chunk_v_offset = input->v_offset; + + tcstream->size += tcstream->chunk_size; + if (tcstream->max_size > 0 && tcstream->size > tcstream->max_size) { + tcstream->error = "Total chunked payload size exceeds maximum"; + stream->istream.stream_errno = EMSGSIZE; + return -1; + } + } return ret; } } @@ -495,11 +504,12 @@ } struct istream * -http_transfer_chunked_istream_create(struct istream *input) +http_transfer_chunked_istream_create(struct istream *input, uoff_t max_size) { struct http_transfer_chunked_istream *tcstream; tcstream = i_new(struct http_transfer_chunked_istream, 1); + tcstream->max_size = max_size; tcstream->istream.max_buffer_size = input->real_stream->max_buffer_size; diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/http-transfer.h --- a/src/lib-http/http-transfer.h Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/http-transfer.h Sun Sep 15 03:52:01 2013 +0300 @@ -18,7 +18,7 @@ // FIXME: we currently lack a means to get error strings from the input stream struct istream * - http_transfer_chunked_istream_create(struct istream *input); +http_transfer_chunked_istream_create(struct istream *input, uoff_t max_size); struct ostream * http_transfer_chunked_ostream_create(struct ostream *output); diff -r 28e5d58e49dc -r a335db9dca6a src/lib-http/test-http-transfer.c --- a/src/lib-http/test-http-transfer.c Sun Sep 15 03:50:08 2013 +0300 +++ b/src/lib-http/test-http-transfer.c Sun Sep 15 03:52:01 2013 +0300 @@ -99,7 +99,7 @@ test_begin(t_strdup_printf("http transfer_chunked input valid [%d]", i)); input = i_stream_create_from_data(in, strlen(in)); - chunked = http_transfer_chunked_istream_create(input); + chunked = http_transfer_chunked_istream_create(input, 0); buffer_set_used_size(payload_buffer, 0); output = o_stream_create_buffer(payload_buffer); @@ -193,7 +193,7 @@ test_begin(t_strdup_printf("http transfer_chunked input invalid [%d]", i)); input = i_stream_create_from_data(in, strlen(in)); - chunked = http_transfer_chunked_istream_create(input); + chunked = http_transfer_chunked_istream_create(input, 0); buffer_set_used_size(payload_buffer, 0); output = o_stream_create_buffer(payload_buffer); @@ -306,7 +306,7 @@ /* create chunked input stream */ input = i_stream_create_from_data (chunked_buffer->data, chunked_buffer->used); - ichunked = http_transfer_chunked_istream_create(input); + ichunked = http_transfer_chunked_istream_create(input, 0); /* read back chunk */ buffer_set_used_size(plain_buffer, 0); From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: Implemented limits on request method and ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7c7e3aa13de7 changeset: 16751:7c7e3aa13de7 user: Stephan Bosch date: Sun Sep 15 03:54:04 2013 +0300 description: lib-http: Implemented limits on request method and target length. diffstat: src/lib-http/http-request-parser.c | 60 +++++++++++++++++++++++++++++++++---- src/lib-http/http-request-parser.h | 19 ++++++----- src/lib-http/http-request.h | 13 ++++++++ src/lib-http/test-http-server.c | 6 +++- 4 files changed, 81 insertions(+), 17 deletions(-) diffs (193 lines): diff -r a335db9dca6a -r 7c7e3aa13de7 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:52:01 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:54:04 2013 +0300 @@ -7,6 +7,8 @@ #include "http-message-parser.h" #include "http-request-parser.h" +#define HTTP_REQUEST_PARSER_MAX_METHOD_LENGTH 32 + enum http_request_parser_state { HTTP_REQUEST_PARSE_STATE_INIT = 0, HTTP_REQUEST_PARSE_STATE_SKIP_LINE, @@ -24,6 +26,8 @@ struct http_message_parser parser; enum http_request_parser_state state; + uoff_t max_target_length; + enum http_request_parse_error error_code; const char *request_method; @@ -34,12 +38,34 @@ struct http_request_parser * http_request_parser_init(struct istream *input, - const struct http_header_limits *hdr_limits) + const struct http_request_limits *limits) { struct http_request_parser *parser; + struct http_header_limits hdr_limits; + uoff_t max_payload_size = limits->max_payload_size; parser = i_new(struct http_request_parser, 1); - http_message_parser_init(&parser->parser, input, hdr_limits, 0); + parser->max_target_length = limits->max_target_length; + + if (limits != NULL) + hdr_limits = limits->header; + else + memset(&hdr_limits, 0, sizeof(hdr_limits)); + + /* substitute default limits */ + if (parser->max_target_length == 0) + parser->max_target_length = HTTP_REQUEST_DEFAULT_MAX_TARGET_LENGTH; + if (hdr_limits.max_size == 0) + hdr_limits.max_size = HTTP_REQUEST_DEFAULT_MAX_HEADER_SIZE; + if (hdr_limits.max_field_size == 0) + hdr_limits.max_field_size = HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELD_SIZE; + if (hdr_limits.max_fields == 0) + hdr_limits.max_fields = HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELDS; + if (max_payload_size == 0) + max_payload_size = HTTP_REQUEST_DEFAULT_MAX_PAYLOAD_SIZE; + + http_message_parser_init + (&parser->parser, input, &hdr_limits, max_payload_size); return parser; } @@ -69,6 +95,11 @@ while (p < parser->parser.end && http_char_is_token(*p)) p++; + if ((p - parser->parser.cur) > HTTP_REQUEST_PARSER_MAX_METHOD_LENGTH) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG; + parser->parser.error = "HTTP request method is too long"; + return -1; + } if (p == parser->parser.end) return 0; parser->request_method = @@ -79,19 +110,32 @@ static int http_request_parse_target(struct http_request_parser *parser) { + struct http_message_parser *_parser = &parser->parser; const unsigned char *p = parser->parser.cur; /* We'll just parse anything up to the first SP or a control char. We could also implement workarounds for buggy HTTP clients and parse anything up to the HTTP-version and return 301 with the - target properly encoded. */ - while (p < parser->parser.end && *p > ' ') + target properly encoded (FIXME). */ + while (p < _parser->end && *p > ' ') p++; - if (p == parser->parser.end) + /* target is too long when explicit limit is exceeded or when input buffer + runs out of space */ + /* FIXME: put limit on full request line rather than target and method + separately */ + /* FIXME: is it wise to keep target in stream buffer? It can become very + large for some applications, increasing the stream buffer size */ + if ((uoff_t)(p - _parser->cur) > parser->max_target_length || + (p == _parser->end && ((uoff_t)(p - _parser->cur) >= + i_stream_get_max_buffer_size(_parser->input)))) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG; + parser->parser.error = "HTTP request target is too long"; + return -1; + } + if (p == _parser->end) return 0; - parser->request_target = - p_strdup_until(parser->parser.msg.pool, parser->parser.cur, p); + parser->request_target = p_strdup_until(_parser->msg.pool, _parser->cur, p); parser->parser.cur = p; return 1; } @@ -272,6 +316,8 @@ return HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; case HTTP_MESSAGE_PARSE_ERROR_NOT_IMPLEMENTED: return HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED; + case HTTP_MESSAGE_PARSE_ERROR_PAYLOAD_TOO_LARGE: + return HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE; case HTTP_MESSAGE_PARSE_ERROR_BROKEN_MESSAGE: return HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; default: diff -r a335db9dca6a -r 7c7e3aa13de7 src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:52:01 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:54:04 2013 +0300 @@ -4,19 +4,20 @@ #include "http-request.h" enum http_request_parse_error { - HTTP_REQUEST_PARSE_ERROR_NONE = 0, /* no error */ - HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM, /* stream error */ - HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST, /* unrecoverable generic error */ - HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST, /* recoverable generic error */ - HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature - (recoverable) */ - HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG, /* method too long (fatal) */ - HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG /* target too long (fatal) */ + HTTP_REQUEST_PARSE_ERROR_NONE = 0, /* no error */ + HTTP_REQUEST_PARSE_ERROR_BROKEN_STREAM, /* stream error */ + HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST, /* unrecoverable generic error */ + HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST, /* recoverable generic error */ + HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature + (recoverable) */ + HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG, /* method too long (fatal) */ + HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG, /* target too long (fatal) */ + HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE /* payload too large (fatal) */ }; struct http_request_parser * http_request_parser_init(struct istream *input, - const struct http_header_limits *hdr_limits) ATTR_NULL(2); + const struct http_request_limits *limits) ATTR_NULL(2); void http_request_parser_deinit(struct http_request_parser **_parser); int http_request_parse_next(struct http_request_parser *parser, diff -r a335db9dca6a -r 7c7e3aa13de7 src/lib-http/http-request.h --- a/src/lib-http/http-request.h Sun Sep 15 03:52:01 2013 +0300 +++ b/src/lib-http/http-request.h Sun Sep 15 03:54:04 2013 +0300 @@ -5,6 +5,19 @@ struct http_url; +#define HTTP_REQUEST_DEFAULT_MAX_TARGET_LENGTH (8 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_SIZE (200 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELD_SIZE (8 * 1024) +#define HTTP_REQUEST_DEFAULT_MAX_HEADER_FIELDS 50 +#define HTTP_REQUEST_DEFAULT_MAX_PAYLOAD_SIZE (1 * 1024 * 1024) + +struct http_request_limits { + uoff_t max_target_length; + uoff_t max_payload_size; + + struct http_header_limits header; +}; + enum http_request_target_format { HTTP_REQUEST_TARGET_FORMAT_ORIGIN = 0, HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, diff -r a335db9dca6a -r 7c7e3aa13de7 src/lib-http/test-http-server.c --- a/src/lib-http/test-http-server.c Sun Sep 15 03:52:01 2013 +0300 +++ b/src/lib-http/test-http-server.c Sun Sep 15 03:54:04 2013 +0300 @@ -81,11 +81,15 @@ static void client_init(int fd) { struct client *client; + struct http_request_limits req_limits; + + memset(&req_limits, 0, sizeof(req_limits)); + req_limits.max_target_length = 4096; client = i_new(struct client, 1); connection_init_server(clients, &client->conn, "(http client)", fd, fd); - client->parser = http_request_parser_init(client->conn.input, 0); + client->parser = http_request_parser_init(client->conn.input, &req_limits); } static void client_accept(void *context ATTR_UNUSED) From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: http-request-parser: Added function to te... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e53f5173fa2d changeset: 16752:e53f5173fa2d user: Stephan Bosch date: Sun Sep 15 03:55:11 2013 +0300 description: lib-http: http-request-parser: Added function to test whether payload from previous request is still being parsed. This is needed in the server implementation to check whether a request is completely read. diffstat: src/lib-http/http-request-parser.c | 7 +++++++ src/lib-http/http-request-parser.h | 2 ++ 2 files changed, 9 insertions(+), 0 deletions(-) diffs (27 lines): diff -r 7c7e3aa13de7 -r e53f5173fa2d src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:54:04 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:55:11 2013 +0300 @@ -327,6 +327,13 @@ return HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST; } +bool http_request_parser_pending_payload(struct http_request_parser *parser) +{ + if (parser->parser.payload == NULL) + return FALSE; + return i_stream_have_bytes_left(parser->parser.payload); +} + int http_request_parse_next(struct http_request_parser *parser, pool_t pool, struct http_request *request, enum http_request_parse_error *error_code_r, const char **error_r) diff -r 7c7e3aa13de7 -r e53f5173fa2d src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:54:04 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:55:11 2013 +0300 @@ -24,4 +24,6 @@ pool_t pool, struct http_request *request, enum http_request_parse_error *error_code_r, const char **error_r); +bool http_request_parser_pending_payload(struct http_request_parser *parser); + #endif From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: Added support for parsing Expect: header ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9da90cf29d91 changeset: 16753:9da90cf29d91 user: Stephan Bosch date: Sun Sep 15 03:55:57 2013 +0300 description: lib-http: Added support for parsing Expect: header (currently only accepts `100-continue'). diffstat: src/lib-http/http-request-parser.c | 160 +++++++++++++++++++++++++++++++++++++ src/lib-http/http-request-parser.h | 2 + src/lib-http/http-request.h | 1 + 3 files changed, 163 insertions(+), 0 deletions(-) diffs (200 lines): diff -r e53f5173fa2d -r 9da90cf29d91 src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Sun Sep 15 03:55:11 2013 +0300 +++ b/src/lib-http/http-request-parser.c Sun Sep 15 03:55:57 2013 +0300 @@ -334,6 +334,159 @@ return i_stream_have_bytes_left(parser->parser.payload); } +static int +http_request_parse_expect_header(struct http_request_parser *parser, + struct http_request *request, const struct http_header_field *hdr) +{ + struct http_message_parser *_parser = &parser->parser; + struct http_parser hparser; + bool parse_error = FALSE; + unsigned int num_expectations = 0; + + /* Expect = 1#expectation + expectation = expect-name [ BWS "=" BWS expect-value ] + *( OWS ";" [ OWS expect-param ] ) + expect-param = expect-name [ BWS "=" BWS expect-value ] + expect-name = token + expect-value = token / quoted-string + */ + http_parser_init(&hparser, (const unsigned char *)hdr->value, hdr->size); + while (!parse_error) { + const char *expect_name, *expect_value; + + /* expect-name */ + if (http_parse_token(&hparser, &expect_name) > 0) { + num_expectations++; + if (strcasecmp(expect_name, "100-continue") == 0) { + request->expect_100_continue = TRUE; + } else { + /* http://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-23 + Section 5.1.1: + + If all received Expect header field(s) are syntactically valid but + contain an expectation that the recipient does not understand or + cannot comply with, the recipient MUST respond with a 417 + (Expectation Failed) status code. A recipient of a syntactically + invalid Expectation header field MUST respond with a 4xx status code + other than 417. + + --> Must check rest of expect header syntax before returning error. + */ + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Unknown Expectation `%s'", expect_name); + } + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end) + break; + + if (*hparser.cur == '=') { + hparser.cur++; + http_parse_ows(&hparser); + + /* value */ + if (http_parse_word(&hparser, &expect_value) <= 0) { + parse_error = TRUE; + break; + } + + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Expectation `%s' has unexpected value", expect_name); + } + } + + /* *( OWS ";" [ OWS expect-param ] ) */ + while (!parse_error) { + const char *attribute, *value; + + /* OWS ";" */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ';') + break; + hparser.cur++; + http_parse_ows(&hparser); + + /* expect-param */ + if (http_parse_token(&hparser, &attribute) <= 0) { + parse_error = TRUE; + break; + } + + /* BWS "=" BWS */ + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != '=') { + parse_error = TRUE; + break; + } + hparser.cur++; + http_parse_ows(&hparser); + + /* value */ + if (http_parse_word(&hparser, &value) <= 0) { + parse_error = TRUE; + break; + } + + if (parser->error_code == HTTP_REQUEST_PARSE_ERROR_NONE) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED; + _parser->error = t_strdup_printf + ("Expectation `%s' has unknown parameter `'%s'", + expect_name, attribute); + } + } + if (parse_error) + break; + } + http_parse_ows(&hparser); + if (hparser.cur >= hparser.end || *hparser.cur != ',') + break; + hparser.cur++; + http_parse_ows(&hparser); + } + + if (parse_error || hparser.cur < hparser.end) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + _parser->error = "Invalid Expect header"; + return -1; + } + + if (parser->error_code != HTTP_REQUEST_PARSE_ERROR_NONE) + return -1; + + if (num_expectations == 0) { + parser->error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST; + _parser->error = "Empty Expect header"; + return -1; + } + return 0; +} + +static int +http_request_parse_headers(struct http_request_parser *parser, + struct http_request *request) +{ + const ARRAY_TYPE(http_header_field) *hdrs; + const struct http_header_field *hdr; + + hdrs = http_header_get_fields(parser->parser.msg.header); + array_foreach(hdrs, hdr) { + int ret = 0; + + if (http_header_field_is(hdr, "Expect")) + ret = http_request_parse_expect_header(parser, request, hdr); + + if (ret < 0) + return -1; + } + return 0; +} + int http_request_parse_next(struct http_request_parser *parser, pool_t pool, struct http_request *request, enum http_request_parse_error *error_code_r, const char **error_r) @@ -421,6 +574,13 @@ return -1; } + /* parse request-specific headers */ + if (http_request_parse_headers(parser, request) < 0) { + *error_code_r = parser->error_code; + *error_r = parser->parser.error; + return -1; + } + request->method = parser->request_method; request->target_raw = parser->request_target; request->version_major = parser->parser.msg.version_major; diff -r e53f5173fa2d -r 9da90cf29d91 src/lib-http/http-request-parser.h --- a/src/lib-http/http-request-parser.h Sun Sep 15 03:55:11 2013 +0300 +++ b/src/lib-http/http-request-parser.h Sun Sep 15 03:55:57 2013 +0300 @@ -10,6 +10,8 @@ HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST, /* recoverable generic error */ HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED, /* used unimplemented feature (recoverable) */ + HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED, /* unknown item in Expect: + header (recoverable) */ HTTP_REQUEST_PARSE_ERROR_METHOD_TOO_LONG, /* method too long (fatal) */ HTTP_REQUEST_PARSE_ERROR_TARGET_TOO_LONG, /* target too long (fatal) */ HTTP_REQUEST_PARSE_ERROR_PAYLOAD_TOO_LARGE /* payload too large (fatal) */ diff -r e53f5173fa2d -r 9da90cf29d91 src/lib-http/http-request.h --- a/src/lib-http/http-request.h Sun Sep 15 03:55:11 2013 +0300 +++ b/src/lib-http/http-request.h Sun Sep 15 03:55:57 2013 +0300 @@ -46,6 +46,7 @@ ARRAY_TYPE(const_string) connection_options; unsigned int connection_close:1; + unsigned int expect_100_continue:1; }; static inline bool From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: Added support for handling HTTP/1.0 messa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ccd39339596e changeset: 16754:ccd39339596e user: Stephan Bosch date: Sun Sep 15 03:56:25 2013 +0300 description: lib-http: Added support for handling HTTP/1.0 messages explicitly. This means that default connection persistence semantics are inverted for HTTP/1.0 and keep-alive Connection option is recognized. diffstat: src/lib-http/http-message-parser.c | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-) diffs (32 lines): diff -r 9da90cf29d91 -r ccd39339596e src/lib-http/http-message-parser.c --- a/src/lib-http/http-message-parser.c Sun Sep 15 03:55:57 2013 +0300 +++ b/src/lib-http/http-message-parser.c Sun Sep 15 03:56:25 2013 +0300 @@ -308,6 +308,7 @@ int http_message_parse_headers(struct http_message_parser *parser) { const unsigned char *field_data; + struct http_message *msg = &parser->msg; const char *field_name, *error; size_t field_size; int ret; @@ -320,6 +321,20 @@ &field_name, &field_data, &field_size, &error)) > 0) { if (field_name == NULL) { /* EOH */ + + /* handle HTTP/1.0 persistence */ + if (msg->version_major == 1 && msg->version_minor == 0 && + !msg->connection_close) { + const char *const *option; + + msg->connection_close = TRUE; + array_foreach(&msg->connection_options, option) { + if (strcasecmp(*option, "Keep-Alive") == 0) { + msg->connection_close = FALSE; + break; + } + } + } return 1; } From dovecot at dovecot.org Sun Sep 15 03:57:22 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 15 Sep 2013 03:57:22 +0300 Subject: dovecot-2.2: lib-http: http-client: Implemented explicit HTTP/1.... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bcaee8677d7d changeset: 16755:bcaee8677d7d user: Stephan Bosch date: Sun Sep 15 03:56:47 2013 +0300 description: lib-http: http-client: Implemented explicit HTTP/1.0 support. diffstat: src/lib-http/http-client-request.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (31 lines): diff -r ccd39339596e -r bcaee8677d7d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Sep 15 03:56:25 2013 +0300 +++ b/src/lib-http/http-client-request.c Sun Sep 15 03:56:47 2013 +0300 @@ -151,9 +151,9 @@ const char *key, const char *value) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); - /* don't allow setting Date header directly; + /* don't allow setting Date or Connection header directly; this is ignored for now for backwards compatibility */ - if (strcasecmp(key, "Date") == 0) + if (strcasecmp(key, "Date") == 0 || strcasecmp(key, "Connection") == 0) return; str_printfa(req->headers, "%s: %s\r\n", key, value); } @@ -423,6 +423,7 @@ str_append(rtext, "Expect: 100-continue\r\n"); } if (req->payload_chunked) { + // FIXME: can't do this for a HTTP/1.0 server str_append(rtext, "Transfer-Encoding: chunked\r\n"); req->payload_output = http_transfer_chunked_ostream_create(output); @@ -434,6 +435,7 @@ req->payload_output = output; o_stream_ref(output); } + str_append(rtext, "Connection: Keep-Alive\r\n"); iov[0].iov_base = str_data(rtext); iov[0].iov_len = str_len(rtext); From pigeonhole at rename-it.nl Sun Sep 15 13:19:19 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:19:19 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: spamtest virustest extensions... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/c20d078d0cf4 changeset: 1695:c20d078d0cf4 user: Stephan Bosch date: Sun Sep 15 12:19:10 2013 +0200 description: lib-sieve: spamtest virustest extensions: Fixed end-of-string testing in configuration parser. diffstat: src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e57214510465 -r c20d078d0cf4 src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c --- a/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Thu Aug 15 21:02:37 2013 +0200 +++ b/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Sun Sep 15 12:19:10 2013 +0200 @@ -122,7 +122,7 @@ spec->header_name = p_strdup_until(pool, data, p); while ( *p == ' ' || *p == '\t' ) p++; - if ( p == '\0' ) { + if ( *p == '\0' ) { spec->regexp_match = FALSE; return TRUE; } From pigeonhole at rename-it.nl Sun Sep 15 13:24:57 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:24:57 +0200 Subject: dovecot-2.0-pigeonhole: lib-sieve: spamtest virustest extensions... Message-ID: details: http://hg.rename-it.nl/dovecot-2.0-pigeonhole/rev/27285d227485 changeset: 1561:27285d227485 user: Stephan Bosch date: Sun Sep 15 12:19:10 2013 +0200 description: lib-sieve: spamtest virustest extensions: Fixed end-of-string testing in configuration parser. diffstat: src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 5e61de7d3196 -r 27285d227485 src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c --- a/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Fri May 24 13:07:23 2013 +0200 +++ b/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Sun Sep 15 12:19:10 2013 +0200 @@ -122,7 +122,7 @@ spec->header_name = p_strdup_until(pool, data, p); while ( *p == ' ' || *p == '\t' ) p++; - if ( p == '\0' ) { + if ( *p == '\0' ) { spec->regexp_match = FALSE; return TRUE; } From pigeonhole at rename-it.nl Sun Sep 15 13:41:42 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:41:42 +0200 Subject: dovecot-1.2-sieve: lib-sieve: spamtest virustest extensions: Fix... Message-ID: details: http://hg.rename-it.nl/dovecot-1.2-sieve/rev/494d51a4b07b changeset: 1292:494d51a4b07b user: Stephan Bosch date: Sun Sep 15 12:19:10 2013 +0200 description: lib-sieve: spamtest virustest extensions: Fixed end-of-string testing in configuration parser. diffstat: src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3adf9046ded8 -r 494d51a4b07b src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c --- a/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Thu May 19 23:41:02 2011 +0200 +++ b/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Sun Sep 15 12:19:10 2013 +0200 @@ -121,7 +121,7 @@ spec->header_name = p_strdup_until(pool, data, p); while ( *p == ' ' || *p == '\t' ) p++; - if ( p == '\0' ) { + if ( *p == '\0' ) { spec->regexp_match = FALSE; return TRUE; } From pigeonhole at rename-it.nl Sun Sep 15 13:47:38 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:47:38 +0200 Subject: dovecot-2.2-pigeonhole: Merged changes from Pigeonhole 0.3 tree. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/78215acd5614 changeset: 1796:78215acd5614 user: Stephan Bosch date: Sun Sep 15 12:43:25 2013 +0200 description: Merged changes from Pigeonhole 0.3 tree. diffstat: src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 00fbc2bc1fad -r 78215acd5614 src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c --- a/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Wed Sep 04 18:12:44 2013 +0200 +++ b/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Sun Sep 15 12:43:25 2013 +0200 @@ -122,7 +122,7 @@ spec->header_name = p_strdup_until(pool, data, p); while ( *p == ' ' || *p == '\t' ) p++; - if ( p == '\0' ) { + if ( *p == '\0' ) { spec->regexp_match = FALSE; return TRUE; } From pigeonhole at rename-it.nl Sun Sep 15 13:47:38 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:47:38 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: spamtest virustest extensions... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c20d078d0cf4 changeset: 1795:c20d078d0cf4 user: Stephan Bosch date: Sun Sep 15 12:19:10 2013 +0200 description: lib-sieve: spamtest virustest extensions: Fixed end-of-string testing in configuration parser. diffstat: src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e57214510465 -r c20d078d0cf4 src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c --- a/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Thu Aug 15 21:02:37 2013 +0200 +++ b/src/lib-sieve/plugins/spamvirustest/ext-spamvirustest-common.c Sun Sep 15 12:19:10 2013 +0200 @@ -122,7 +122,7 @@ spec->header_name = p_strdup_until(pool, data, p); while ( *p == ' ' || *p == '\t' ) p++; - if ( p == '\0' ) { + if ( *p == '\0' ) { spec->regexp_match = FALSE; return TRUE; } From pigeonhole at rename-it.nl Sun Sep 15 13:47:38 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 12:47:38 +0200 Subject: dovecot-2.2-pigeonhole: doveadm-sieve plugin: Fixed segfault bug. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8be59d267be3 changeset: 1797:8be59d267be3 user: Stephan Bosch date: Sun Sep 15 12:47:33 2013 +0200 description: doveadm-sieve plugin: Fixed segfault bug. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-plugin.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 78215acd5614 -r 8be59d267be3 src/plugins/doveadm-sieve/doveadm-sieve-plugin.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Sun Sep 15 12:43:25 2013 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Sun Sep 15 12:47:33 2013 +0200 @@ -145,8 +145,6 @@ if (mailbox_attribute_value_to_string(storage, value, &scriptname) < 0) return -1; - i_assert(scriptname[0] == MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_LINK); - scriptname++; if (scriptname == NULL) { /* don't affect non-link active script */ @@ -167,6 +165,8 @@ } return 0; } + i_assert(scriptname[0] == MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_LINK); + scriptname++; /* activate specified script */ script = sieve_storage_script_init(svstorage, scriptname); From pigeonhole at rename-it.nl Sun Sep 15 18:48:31 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 17:48:31 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: Fixed code block read bounds ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/0ad4ca692650 changeset: 1696:0ad4ca692650 user: Stephan Bosch date: Sun Sep 15 17:47:53 2013 +0200 description: lib-sieve: Fixed code block read bounds checking. diffstat: src/lib-sieve/sieve-binary-code.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r c20d078d0cf4 -r 0ad4ca692650 src/lib-sieve/sieve-binary-code.c --- a/src/lib-sieve/sieve-binary-code.c Sun Sep 15 12:19:10 2013 +0200 +++ b/src/lib-sieve/sieve-binary-code.c Sun Sep 15 17:47:53 2013 +0200 @@ -214,7 +214,7 @@ ((const int8_t *) (&_code[*address])) #define ADDR_BYTES_LEFT(address) \ - ((_code_size) - (*address)) + ((*address) > _code_size ? 0 : ((_code_size) - (*address))) #define ADDR_JUMP(address, offset) \ (*address) += offset @@ -350,7 +350,7 @@ ADDR_CODE_READ(sblock); - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return FALSE; (*offset_r) = code = ADDR_DATA_AT(address); @@ -382,7 +382,7 @@ if ( objs->count == 1 ) return objs->objects; - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return NULL; code = ADDR_DATA_AT(address); From pigeonhole at rename-it.nl Sun Sep 15 18:52:05 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 15 Sep 2013 17:52:05 +0200 Subject: dovecot-1.2-sieve: lib-sieve: Fixed code block read bounds check... Message-ID: details: http://hg.rename-it.nl/dovecot-1.2-sieve/rev/885bb670a0f5 changeset: 1293:885bb670a0f5 user: Stephan Bosch date: Sun Sep 15 17:51:59 2013 +0200 description: lib-sieve: Fixed code block read bounds checking. diffstat: src/lib-sieve/sieve-binary.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r 494d51a4b07b -r 885bb670a0f5 src/lib-sieve/sieve-binary.c --- a/src/lib-sieve/sieve-binary.c Sun Sep 15 12:19:10 2013 +0200 +++ b/src/lib-sieve/sieve-binary.c Sun Sep 15 17:51:59 2013 +0200 @@ -1637,7 +1637,7 @@ #define ADDR_POINTER(binary, address) \ ((uint8_t *) (&(binary)->code[*address])) #define ADDR_BYTES_LEFT(binary, address) \ - ((binary)->code_size - (*address)) + ((*address) > (binary)->code_size ? 0 : ((binary)->code_size - (*address))) #define ADDR_JUMP(address, offset) \ (*address) += offset @@ -1761,7 +1761,7 @@ unsigned int offset = *offset_r; const struct sieve_extension *ext = NULL; - if ( ADDR_BYTES_LEFT(sbin, address) <= 0 ) + if ( ADDR_BYTES_LEFT(sbin, address) == 0 ) return FALSE; (*offset_r) = code = ADDR_DATA_AT(sbin, address); @@ -1791,7 +1791,7 @@ if ( objs->count == 1 ) return objs->objects; - if ( ADDR_BYTES_LEFT(sbin, address) <= 0 ) + if ( ADDR_BYTES_LEFT(sbin, address) == 0 ) return NULL; code = ADDR_DATA_AT(sbin, address); From dovecot at dovecot.org Mon Sep 16 01:02:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Sep 2013 01:02:20 +0300 Subject: dovecot-2.2: lib-http: http-client: Fixed segfault caused by ear... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7fc1dcfc5e3a changeset: 16756:7fc1dcfc5e3a user: Stephan Bosch date: Mon Sep 16 01:02:03 2013 +0300 description: lib-http: http-client: Fixed segfault caused by earlier improvement of connection output locking. Segfault was triggered when an aborted request received a response. diffstat: src/lib-http/http-client-connection.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r bcaee8677d7d -r 7fc1dcfc5e3a src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sun Sep 15 03:56:47 2013 +0300 +++ b/src/lib-http/http-client-connection.c Mon Sep 16 01:02:03 2013 +0300 @@ -575,6 +575,10 @@ "Got %u response for request %s", response.status, http_client_request_label(req)); + /* make sure connection output is unlocked if 100-continue failed */ + if (req->payload_sync && !conn->payload_continue) + conn->output_locked = FALSE; + /* remove request from queue */ array_delete(&conn->request_wait_list, 0, 1); aborted = (req->state == HTTP_REQUEST_STATE_ABORTED); @@ -582,8 +586,6 @@ http_client_request_unref(&req); conn->close_indicated = response.connection_close; - if (req->payload_sync && !conn->payload_continue) - conn->output_locked = FALSE; if (!aborted) { if (response.status == 417 && req->payload_sync) { From pigeonhole at rename-it.nl Mon Sep 16 01:44:36 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 00:44:36 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: variables extension: fixed da... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/2b7472f55ec5 changeset: 1697:2b7472f55ec5 user: Stephan Bosch date: Mon Sep 16 00:43:55 2013 +0200 description: lib-sieve: variables extension: fixed data stack problem in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 74 +++++++++++++++--------------- 1 files changed, 36 insertions(+), 38 deletions(-) diffs (98 lines): diff -r 0ad4ca692650 -r 2b7472f55ec5 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Sep 15 17:47:53 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:43:55 2013 +0200 @@ -315,58 +315,56 @@ if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - T_BEGIN { - /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; + /* Apply modifiers if necessary (sorted during code generation already) */ + if ( str_len(value) > 0 ) { + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; + ret = SIEVE_EXEC_FAILURE; break; } - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } + sieve_runtime_trace_here + (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", + sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); - sieve_runtime_trace_here - (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", - sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); + value = new_value; + if ( value == NULL ) + break; - value = new_value; - if ( value == NULL ) - break; - - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } } + } - /* Actually assign the value if all is well */ - if ( value != NULL ) { - if ( !sieve_variable_assign(storage, var_index, value) ) - ret = SIEVE_EXEC_BIN_CORRUPT; - else { - if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { - const char *var_name, *var_id; + /* Actually assign the value if all is well */ + if ( value != NULL ) { + if ( !sieve_variable_assign(storage, var_index, value) ) + ret = SIEVE_EXEC_BIN_CORRUPT; + else { + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + const char *var_name, *var_id; - (void)sieve_variable_get_identifier(storage, var_index, &var_name); - var_id = sieve_variable_get_varid(storage, var_index); + (void)sieve_variable_get_identifier(storage, var_index, &var_name); + var_id = sieve_variable_get_varid(storage, var_index); - sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", - var_name, var_id, str_c(value)); - } + sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", + var_name, var_id, str_c(value)); } } - } T_END; + } if ( ret <= 0 ) return ret; if ( value == NULL ) return SIEVE_EXEC_FAILURE; From pigeonhole at rename-it.nl Mon Sep 16 01:47:06 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 00:47:06 +0200 Subject: dovecot-2.1-pigeonhole: sieve-dump tool: Fixed hex output. Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/5b4b8d0cf0f3 changeset: 1698:5b4b8d0cf0f3 user: Stephan Bosch date: Mon Sep 16 00:46:21 2013 +0200 description: sieve-dump tool: Fixed hex output. Messed up signed/unsigned characters. diffstat: src/lib-sieve/sieve-binary-dumper.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diffs (38 lines): diff -r 2b7472f55ec5 -r 5b4b8d0cf0f3 src/lib-sieve/sieve-binary-dumper.c --- a/src/lib-sieve/sieve-binary-dumper.c Mon Sep 16 00:43:55 2013 +0200 +++ b/src/lib-sieve/sieve-binary-dumper.c Mon Sep 16 00:46:21 2013 +0200 @@ -218,10 +218,10 @@ buffer_t *blockbuf = sieve_binary_block_get_buffer(sblock); string_t *line; size_t data_size; - const char *data; + const unsigned char *data; size_t offset; - data = (const char *) buffer_get_data(blockbuf, &data_size); + data = buffer_get_data(blockbuf, &data_size); // FIXME: calculate offset more nicely. sieve_binary_dump_sectionf @@ -237,7 +237,7 @@ str_printfa(line, "%08llx ", (unsigned long long) offset); for ( b = 0; b < len; b++ ) { - str_printfa(line, "%02x ", (unsigned int) data[offset+b]); + str_printfa(line, "%02x ", data[offset+b]); if ( b == 7 ) str_append_c(line, ' '); } @@ -252,10 +252,10 @@ str_append(line, " |"); for ( b = 0; b < len; b++ ) { - const char c = data[offset+b]; + const unsigned char c = data[offset+b]; if ( c >= 32 && c <= 126 ) - str_append_c(line, c); + str_append_c(line, (const char)c); else str_append_c(line, '.'); } From pigeonhole at rename-it.nl Mon Sep 16 01:56:10 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 00:56:10 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: variables extension: Fixed co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/aadc77880254 changeset: 1699:aadc77880254 user: Stephan Bosch date: Mon Sep 16 00:55:29 2013 +0200 description: lib-sieve: variables extension: Fixed code corruption bug in 'set' command. Bug whould occur when modifier was used and the value became "" at some point. In that case not all operands (modifiers) were read, causing the subsequent operation to start at an erroneous position. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 18 +++++++++--------- 1 files changed, 9 insertions(+), 9 deletions(-) diffs (30 lines): diff -r 5b4b8d0cf0f3 -r aadc77880254 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:46:21 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:55:29 2013 +0200 @@ -316,17 +316,17 @@ str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { - value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; - break; - } + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + if ( str_len(value) > 0 ) { if ( modf.def != NULL && modf.def->modify != NULL ) { if ( !modf.def->modify(value, &new_value) ) { value = NULL; From pigeonhole at rename-it.nl Mon Sep 16 02:14:25 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:14:25 +0200 Subject: dovecot-1.2-sieve: lib-sieve: variables extension: fixed data st... Message-ID: details: http://hg.rename-it.nl/dovecot-1.2-sieve/rev/d5188ad86a3d changeset: 1294:d5188ad86a3d user: Stephan Bosch date: Mon Sep 16 01:02:34 2013 +0200 description: lib-sieve: variables extension: fixed data stack problem in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 64 +++++++++++++++--------------- 1 files changed, 31 insertions(+), 33 deletions(-) diffs (80 lines): diff -r 885bb670a0f5 -r d5188ad86a3d src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Sep 15 17:51:59 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 01:02:34 2013 +0200 @@ -320,45 +320,43 @@ if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - T_BEGIN { - /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; - - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + /* Apply modifiers if necessary (sorted during code generation already) */ + if ( str_len(value) > 0 ) { + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; + + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + + sieve_runtime_trace_error(renv, "invalid modifier operand"); + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { value = NULL; - - sieve_runtime_trace_error(renv, "invalid modifier operand"); - ret = SIEVE_EXEC_BIN_CORRUPT; + ret = SIEVE_EXEC_FAILURE; break; } - - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } - value = new_value; - if ( value == NULL ) - break; + value = new_value; + if ( value == NULL ) + break; - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } - } - - /* Actually assign the value if all is well */ - if ( value != NULL ) { - if ( !sieve_variable_assign(storage, var_index, value) ) - ret = SIEVE_EXEC_BIN_CORRUPT; - } - } T_END; + } + } + + /* Actually assign the value if all is well */ + if ( value != NULL ) { + if ( !sieve_variable_assign(storage, var_index, value) ) + ret = SIEVE_EXEC_BIN_CORRUPT; + } if ( ret <= 0 ) return ret; From pigeonhole at rename-it.nl Mon Sep 16 02:14:25 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:14:25 +0200 Subject: dovecot-1.2-sieve: lib-sieve: variables extension: Fixed code co... Message-ID: details: http://hg.rename-it.nl/dovecot-1.2-sieve/rev/484b6f13131c changeset: 1295:484b6f13131c user: Stephan Bosch date: Mon Sep 16 01:14:17 2013 +0200 description: lib-sieve: variables extension: Fixed code corruption bug in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 24 ++++++++++++------------ 1 files changed, 12 insertions(+), 12 deletions(-) diffs (35 lines): diff -r d5188ad86a3d -r 484b6f13131c src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 01:02:34 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 01:14:17 2013 +0200 @@ -321,19 +321,19 @@ str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; - - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { - value = NULL; + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; + + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; - sieve_runtime_trace_error(renv, "invalid modifier operand"); - ret = SIEVE_EXEC_BIN_CORRUPT; - break; - } - + sieve_runtime_trace_error(renv, "invalid modifier operand"); + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( str_len(value) > 0 ) { if ( modf.def != NULL && modf.def->modify != NULL ) { if ( !modf.def->modify(value, &new_value) ) { value = NULL; From pigeonhole at rename-it.nl Mon Sep 16 02:17:30 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:17:30 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: variables extension: Fixed co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/aadc77880254 changeset: 1801:aadc77880254 user: Stephan Bosch date: Mon Sep 16 00:55:29 2013 +0200 description: lib-sieve: variables extension: Fixed code corruption bug in 'set' command. Bug whould occur when modifier was used and the value became "" at some point. In that case not all operands (modifiers) were read, causing the subsequent operation to start at an erroneous position. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 18 +++++++++--------- 1 files changed, 9 insertions(+), 9 deletions(-) diffs (30 lines): diff -r 5b4b8d0cf0f3 -r aadc77880254 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:46:21 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:55:29 2013 +0200 @@ -316,17 +316,17 @@ str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { - value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; - break; - } + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + if ( str_len(value) > 0 ) { if ( modf.def != NULL && modf.def->modify != NULL ) { if ( !modf.def->modify(value, &new_value) ) { value = NULL; From pigeonhole at rename-it.nl Mon Sep 16 02:17:30 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:17:30 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: variables extension: fixed da... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/2b7472f55ec5 changeset: 1799:2b7472f55ec5 user: Stephan Bosch date: Mon Sep 16 00:43:55 2013 +0200 description: lib-sieve: variables extension: fixed data stack problem in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 74 +++++++++++++++--------------- 1 files changed, 36 insertions(+), 38 deletions(-) diffs (98 lines): diff -r 0ad4ca692650 -r 2b7472f55ec5 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Sep 15 17:47:53 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 00:43:55 2013 +0200 @@ -315,58 +315,56 @@ if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - T_BEGIN { - /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; + /* Apply modifiers if necessary (sorted during code generation already) */ + if ( str_len(value) > 0 ) { + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; + ret = SIEVE_EXEC_FAILURE; break; } - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } + sieve_runtime_trace_here + (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", + sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); - sieve_runtime_trace_here - (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", - sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); + value = new_value; + if ( value == NULL ) + break; - value = new_value; - if ( value == NULL ) - break; - - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } } + } - /* Actually assign the value if all is well */ - if ( value != NULL ) { - if ( !sieve_variable_assign(storage, var_index, value) ) - ret = SIEVE_EXEC_BIN_CORRUPT; - else { - if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { - const char *var_name, *var_id; + /* Actually assign the value if all is well */ + if ( value != NULL ) { + if ( !sieve_variable_assign(storage, var_index, value) ) + ret = SIEVE_EXEC_BIN_CORRUPT; + else { + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + const char *var_name, *var_id; - (void)sieve_variable_get_identifier(storage, var_index, &var_name); - var_id = sieve_variable_get_varid(storage, var_index); + (void)sieve_variable_get_identifier(storage, var_index, &var_name); + var_id = sieve_variable_get_varid(storage, var_index); - sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", - var_name, var_id, str_c(value)); - } + sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", + var_name, var_id, str_c(value)); } } - } T_END; + } if ( ret <= 0 ) return ret; if ( value == NULL ) return SIEVE_EXEC_FAILURE; From pigeonhole at rename-it.nl Mon Sep 16 02:17:30 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:17:30 +0200 Subject: dovecot-2.2-pigeonhole: Merged changes from v0.3 tree. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b1db52d0c0b3 changeset: 1802:b1db52d0c0b3 user: Stephan Bosch date: Mon Sep 16 01:15:55 2013 +0200 description: Merged changes from v0.3 tree. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 74 +++++++++++++++--------------- src/lib-sieve/sieve-binary-code.c | 6 +- src/lib-sieve/sieve-binary-dumper.c | 10 ++-- 3 files changed, 44 insertions(+), 46 deletions(-) diffs (166 lines): diff -r 8be59d267be3 -r b1db52d0c0b3 src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Sep 15 12:47:33 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Mon Sep 16 01:15:55 2013 +0200 @@ -315,58 +315,56 @@ if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - T_BEGIN { - /* Apply modifiers if necessary (sorted during code generation already) */ + /* Apply modifiers if necessary (sorted during code generation already) */ + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; + + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; - - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; + ret = SIEVE_EXEC_FAILURE; break; } - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } + sieve_runtime_trace_here + (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", + sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); - sieve_runtime_trace_here - (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", - sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); + value = new_value; + if ( value == NULL ) + break; - value = new_value; - if ( value == NULL ) - break; - - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } } + } - /* Actually assign the value if all is well */ - if ( value != NULL ) { - if ( !sieve_variable_assign(storage, var_index, value) ) - ret = SIEVE_EXEC_BIN_CORRUPT; - else { - if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { - const char *var_name, *var_id; + /* Actually assign the value if all is well */ + if ( value != NULL ) { + if ( !sieve_variable_assign(storage, var_index, value) ) + ret = SIEVE_EXEC_BIN_CORRUPT; + else { + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + const char *var_name, *var_id; - (void)sieve_variable_get_identifier(storage, var_index, &var_name); - var_id = sieve_variable_get_varid(storage, var_index); + (void)sieve_variable_get_identifier(storage, var_index, &var_name); + var_id = sieve_variable_get_varid(storage, var_index); - sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", - var_name, var_id, str_c(value)); - } + sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", + var_name, var_id, str_c(value)); } } - } T_END; + } if ( ret <= 0 ) return ret; if ( value == NULL ) return SIEVE_EXEC_FAILURE; diff -r 8be59d267be3 -r b1db52d0c0b3 src/lib-sieve/sieve-binary-code.c --- a/src/lib-sieve/sieve-binary-code.c Sun Sep 15 12:47:33 2013 +0200 +++ b/src/lib-sieve/sieve-binary-code.c Mon Sep 16 01:15:55 2013 +0200 @@ -214,7 +214,7 @@ ((const int8_t *) (&_code[*address])) #define ADDR_BYTES_LEFT(address) \ - ((_code_size) - (*address)) + ((*address) > _code_size ? 0 : ((_code_size) - (*address))) #define ADDR_JUMP(address, offset) \ (*address) += offset @@ -350,7 +350,7 @@ ADDR_CODE_READ(sblock); - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return FALSE; (*offset_r) = code = ADDR_DATA_AT(address); @@ -382,7 +382,7 @@ if ( objs->count == 1 ) return objs->objects; - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return NULL; code = ADDR_DATA_AT(address); diff -r 8be59d267be3 -r b1db52d0c0b3 src/lib-sieve/sieve-binary-dumper.c --- a/src/lib-sieve/sieve-binary-dumper.c Sun Sep 15 12:47:33 2013 +0200 +++ b/src/lib-sieve/sieve-binary-dumper.c Mon Sep 16 01:15:55 2013 +0200 @@ -218,10 +218,10 @@ buffer_t *blockbuf = sieve_binary_block_get_buffer(sblock); string_t *line; size_t data_size; - const char *data; + const unsigned char *data; size_t offset; - data = (const char *) buffer_get_data(blockbuf, &data_size); + data = buffer_get_data(blockbuf, &data_size); // FIXME: calculate offset more nicely. sieve_binary_dump_sectionf @@ -237,7 +237,7 @@ str_printfa(line, "%08llx ", (unsigned long long) offset); for ( b = 0; b < len; b++ ) { - str_printfa(line, "%02x ", (unsigned int) data[offset+b]); + str_printfa(line, "%02x ", data[offset+b]); if ( b == 7 ) str_append_c(line, ' '); } @@ -252,10 +252,10 @@ str_append(line, " |"); for ( b = 0; b < len; b++ ) { - const char c = data[offset+b]; + const unsigned char c = data[offset+b]; if ( c >= 32 && c <= 126 ) - str_append_c(line, c); + str_append_c(line, (const char)c); else str_append_c(line, '.'); } From pigeonhole at rename-it.nl Mon Sep 16 02:17:30 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:17:30 +0200 Subject: dovecot-2.2-pigeonhole: sieve-dump tool: Fixed hex output. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5b4b8d0cf0f3 changeset: 1800:5b4b8d0cf0f3 user: Stephan Bosch date: Mon Sep 16 00:46:21 2013 +0200 description: sieve-dump tool: Fixed hex output. Messed up signed/unsigned characters. diffstat: src/lib-sieve/sieve-binary-dumper.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diffs (38 lines): diff -r 2b7472f55ec5 -r 5b4b8d0cf0f3 src/lib-sieve/sieve-binary-dumper.c --- a/src/lib-sieve/sieve-binary-dumper.c Mon Sep 16 00:43:55 2013 +0200 +++ b/src/lib-sieve/sieve-binary-dumper.c Mon Sep 16 00:46:21 2013 +0200 @@ -218,10 +218,10 @@ buffer_t *blockbuf = sieve_binary_block_get_buffer(sblock); string_t *line; size_t data_size; - const char *data; + const unsigned char *data; size_t offset; - data = (const char *) buffer_get_data(blockbuf, &data_size); + data = buffer_get_data(blockbuf, &data_size); // FIXME: calculate offset more nicely. sieve_binary_dump_sectionf @@ -237,7 +237,7 @@ str_printfa(line, "%08llx ", (unsigned long long) offset); for ( b = 0; b < len; b++ ) { - str_printfa(line, "%02x ", (unsigned int) data[offset+b]); + str_printfa(line, "%02x ", data[offset+b]); if ( b == 7 ) str_append_c(line, ' '); } @@ -252,10 +252,10 @@ str_append(line, " |"); for ( b = 0; b < len; b++ ) { - const char c = data[offset+b]; + const unsigned char c = data[offset+b]; if ( c >= 32 && c <= 126 ) - str_append_c(line, c); + str_append_c(line, (const char)c); else str_append_c(line, '.'); } From pigeonhole at rename-it.nl Mon Sep 16 02:17:30 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 16 Sep 2013 01:17:30 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed code block read bounds ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0ad4ca692650 changeset: 1798:0ad4ca692650 user: Stephan Bosch date: Sun Sep 15 17:47:53 2013 +0200 description: lib-sieve: Fixed code block read bounds checking. diffstat: src/lib-sieve/sieve-binary-code.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r c20d078d0cf4 -r 0ad4ca692650 src/lib-sieve/sieve-binary-code.c --- a/src/lib-sieve/sieve-binary-code.c Sun Sep 15 12:19:10 2013 +0200 +++ b/src/lib-sieve/sieve-binary-code.c Sun Sep 15 17:47:53 2013 +0200 @@ -214,7 +214,7 @@ ((const int8_t *) (&_code[*address])) #define ADDR_BYTES_LEFT(address) \ - ((_code_size) - (*address)) + ((*address) > _code_size ? 0 : ((_code_size) - (*address))) #define ADDR_JUMP(address, offset) \ (*address) += offset @@ -350,7 +350,7 @@ ADDR_CODE_READ(sblock); - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return FALSE; (*offset_r) = code = ADDR_DATA_AT(address); @@ -382,7 +382,7 @@ if ( objs->count == 1 ) return objs->objects; - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return NULL; code = ADDR_DATA_AT(address); From dovecot at dovecot.org Mon Sep 16 13:18:09 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 16 Sep 2013 13:18:09 +0300 Subject: dovecot-2.2: Fixed uri_parse_slashslash_authority() to skip over... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a114a8bfce61 changeset: 16757:a114a8bfce61 user: Timo Sirainen date: Mon Sep 16 13:17:52 2013 +0300 description: Fixed uri_parse_slashslash_authority() to skip over "//". diffstat: src/lib/uri-util.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 7fc1dcfc5e3a -r a114a8bfce61 src/lib/uri-util.c --- a/src/lib/uri-util.c Mon Sep 16 01:02:03 2013 +0300 +++ b/src/lib/uri-util.c Mon Sep 16 13:17:52 2013 +0300 @@ -569,6 +569,7 @@ parser->cur[1] != '/') return 0; + parser->cur += 2; return uri_parse_authority(parser, auth); } From dovecot at dovecot.org Tue Sep 17 02:41:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 02:41:36 +0300 Subject: dovecot-2.2: lib-http: http-client: Allow overriding all implici... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/145be4e876a8 changeset: 16758:145be4e876a8 user: Stephan Bosch date: Tue Sep 17 02:40:17 2013 +0300 description: lib-http: http-client: Allow overriding all implicitly generated special headers. Which are: Connection, Content-Length, Date, Expect, Host, and Transfer-Encoding. diffstat: src/lib-http/http-client-private.h | 6 +++ src/lib-http/http-client-request.c | 74 ++++++++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 18 deletions(-) diffs (127 lines): diff -r a114a8bfce61 -r 145be4e876a8 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Mon Sep 16 13:17:52 2013 +0300 +++ b/src/lib-http/http-client-private.h Tue Sep 17 02:40:17 2013 +0300 @@ -69,6 +69,12 @@ enum http_request_state state; + unsigned int have_hdr_connection:1; + unsigned int have_hdr_date:1; + unsigned int have_hdr_expect:1; + unsigned int have_hdr_host:1; + unsigned int have_hdr_body_spec:1; + unsigned int payload_sync:1; unsigned int payload_chunked:1; unsigned int payload_wait:1; diff -r a114a8bfce61 -r 145be4e876a8 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Mon Sep 16 13:17:52 2013 +0300 +++ b/src/lib-http/http-client-request.c Tue Sep 17 02:40:17 2013 +0300 @@ -151,10 +151,31 @@ const char *key, const char *value) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); - /* don't allow setting Date or Connection header directly; - this is ignored for now for backwards compatibility */ - if (strcasecmp(key, "Date") == 0 || strcasecmp(key, "Connection") == 0) - return; + /* mark presence of special headers */ + switch (key[0]) { + case 'c': case 'C': + if (strcasecmp(key, "Connection") == 0) + req->have_hdr_connection = TRUE; + else if (strcasecmp(key, "Content-Length") == 0) + req->have_hdr_body_spec = TRUE; + break; + case 'd': case 'D': + if (strcasecmp(key, "Date") == 0) + req->have_hdr_date = TRUE; + break; + case 'e': case 'E': + if (strcasecmp(key, "Expect") == 0) + req->have_hdr_expect = TRUE; + break; + case 'h': case 'H': + if (strcasecmp(key, "Host") == 0) + req->have_hdr_host = TRUE; + break; + case 't': case 'T': + if (strcasecmp(key, "Transfer-Encoding") == 0) + req->have_hdr_body_spec = TRUE; + break; + } str_printfa(req->headers, "%s: %s\r\n", key, value); } @@ -407,40 +428,57 @@ i_assert(!req->conn->output_locked); i_assert(req->payload_output == NULL); + /* create request line */ str_append(rtext, req->method); str_append(rtext, " "); str_append(rtext, req->target); - str_append(rtext, " HTTP/1.1\r\nHost: "); - str_append(rtext, req->hostname); - if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) || - (req->ssl && req->port != HTTPS_DEFAULT_PORT)) { - str_printfa(rtext, ":%u", req->port); + str_append(rtext, " HTTP/1.1\r\n"); + + /* create special headers implicitly if not set explicitly using + http_client_request_add_header() */ + if (!req->have_hdr_host) { + str_append(rtext, "Host: "); + str_append(rtext, req->hostname); + if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) || + (req->ssl && req->port != HTTPS_DEFAULT_PORT)) { + str_printfa(rtext, ":%u", req->port); + } + str_append(rtext, "\r\n"); } - str_append(rtext, "\r\nDate: "); - str_append(rtext, http_date_create(req->date)); - str_append(rtext, "\r\n"); - if (req->payload_sync) { + if (!req->have_hdr_date) { + str_append(rtext, "Date: "); + str_append(rtext, http_date_create(req->date)); + str_append(rtext, "\r\n"); + } + if (!req->have_hdr_expect && req->payload_sync) { str_append(rtext, "Expect: 100-continue\r\n"); } if (req->payload_chunked) { // FIXME: can't do this for a HTTP/1.0 server - str_append(rtext, "Transfer-Encoding: chunked\r\n"); + if (!req->have_hdr_body_spec) + str_append(rtext, "Transfer-Encoding: chunked\r\n"); req->payload_output = http_transfer_chunked_ostream_create(output); } else if (req->payload_input != NULL) { /* send Content-Length if we have specified a payload, even if it's 0 bytes. */ - str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", - req->payload_size); + if (!req->have_hdr_body_spec) { + str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n", + req->payload_size); + } req->payload_output = output; o_stream_ref(output); } - str_append(rtext, "Connection: Keep-Alive\r\n"); + if (!req->have_hdr_connection) + str_append(rtext, "Connection: Keep-Alive\r\n"); + /* request line + implicit headers */ iov[0].iov_base = str_data(rtext); - iov[0].iov_len = str_len(rtext); + iov[0].iov_len = str_len(rtext); + /* explicit headers */ iov[1].iov_base = str_data(req->headers); iov[1].iov_len = str_len(req->headers); + /* end of header */ iov[2].iov_base = "\r\n"; iov[2].iov_len = 2; From dovecot at dovecot.org Tue Sep 17 02:42:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 02:42:26 +0300 Subject: dovecot-2.2: imap: Fixed/improved error logging for FETCH Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be26ae8a9fca changeset: 16759:be26ae8a9fca user: Timo Sirainen date: Mon Sep 16 10:05:24 2013 +0300 description: imap: Fixed/improved error logging for FETCH diffstat: src/imap/imap-fetch-body.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 145be4e876a8 -r be26ae8a9fca src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Tue Sep 17 02:40:17 2013 +0300 +++ b/src/imap/imap-fetch-body.c Mon Sep 16 10:05:24 2013 +0300 @@ -34,8 +34,8 @@ errno = state->cur_input->stream_errno; mail_storage_set_critical(state->cur_mail->box->storage, "read(%s) failed: %m (FETCH %s for mailbox %s UID %u)", + i_stream_get_name(state->cur_input), state->cur_human_name, - i_stream_get_name(state->cur_input), mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid); } @@ -102,9 +102,10 @@ } if (!i_stream_have_bytes_left(state->cur_input)) { /* Input stream gave less data than expected */ - i_error("FETCH %s for mailbox %s UID %u " + i_error("read(%s): FETCH %s for mailbox %s UID %u " "got too little data: " "%"PRIuUOFF_T" vs %"PRIuUOFF_T, + i_stream_get_name(state->cur_input), state->cur_human_name, mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid, From dovecot at dovecot.org Tue Sep 17 21:58:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 21:58:06 +0300 Subject: dovecot-2.2: lib-imap: imap-url: Forgot to check for the presenc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6be5d8d8af2d changeset: 16760:6be5d8d8af2d user: Stephan Bosch date: Tue Sep 17 21:57:14 2013 +0300 description: lib-imap: imap-url: Forgot to check for the presence of ':' in userinfo, which is not allowed. diffstat: src/lib-imap/imap-url.c | 41 ++++++++++++++++++++++++++++------------- src/lib-imap/test-imap-url.c | 5 +++++ 2 files changed, 33 insertions(+), 13 deletions(-) diffs (99 lines): diff -r be26ae8a9fca -r 6be5d8d8af2d src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Mon Sep 16 10:05:24 2013 +0300 +++ b/src/lib-imap/imap-url.c Tue Sep 17 21:57:14 2013 +0300 @@ -226,34 +226,49 @@ /* iuserinfo = enc-user [iauth] / [enc-user] iauth */ if (auth.enc_userinfo != NULL) { - const char *p; + const char *p, *uend; /* Scan for ";AUTH=" */ - p = strchr(auth.enc_userinfo, ';'); - if (p != NULL) { - if (strncasecmp(p, ";AUTH=",6) != 0) { + for (p = auth.enc_userinfo; *p != '\0'; p++) { + if (*p == ';') + break; + /* check for unallowed userinfo characters */ + if (*p == ':') { + parser->error = t_strdup_printf( + "Stray ':' in userinfo `%s'", auth.enc_userinfo); + return -1; + } + } + + uend = p; + + if (*p == ';') { + if (strncasecmp(p, ";AUTH=", 6) != 0) { parser->error = t_strdup_printf( "Stray ';' in userinfo `%s'", auth.enc_userinfo); return -1; } - if (strchr(p+1, ';') != NULL) { - parser->error = "Stray ';' after `;AUTH='"; - return -1; + for (p += 6; *p != '\0'; p++) { + if (*p == ';' || *p == ':') { + parser->error = t_strdup_printf( + "Stray '%c' in userinfo `%s'", *p, auth.enc_userinfo); + return -1; + } } } /* enc-user */ - if (url != NULL && p != auth.enc_userinfo) { - if (!uri_data_decode(parser, auth.enc_userinfo, p, &data)) + if (url != NULL && uend > auth.enc_userinfo) { + if (!uri_data_decode(parser, auth.enc_userinfo, uend, &data)) return -1; url->userid = p_strdup(parser->pool, data); } /* ( "*" / enc-auth-type ) */ - if (p != NULL) { - p += 6; + if (*uend == ';') { + p = uend + 6; if (*p == '\0') { parser->error = "Empty auth-type value after ';AUTH='"; return -1; @@ -989,10 +1004,10 @@ /* user */ if (url->userid != NULL || url->auth_type != NULL) { if (url->userid != NULL) - uri_append_user_data(urlstr, ";", url->userid); + uri_append_user_data(urlstr, ";:", url->userid); if (url->auth_type != NULL) { str_append(urlstr, ";AUTH="); - uri_append_user_data(urlstr, ";", url->auth_type); + uri_append_user_data(urlstr, ";:", url->auth_type); } str_append_c(urlstr, '@'); } diff -r be26ae8a9fca -r 6be5d8d8af2d src/lib-imap/test-imap-url.c --- a/src/lib-imap/test-imap-url.c Mon Sep 16 10:05:24 2013 +0300 +++ b/src/lib-imap/test-imap-url.c Tue Sep 17 21:57:14 2013 +0300 @@ -759,6 +759,10 @@ },{ .url = "imap://user;AUTH=@example.com" },{ + .url = "imap://user:password at example.com" + },{ + .url = "imap://user;AUTH=A:B at example.com" + },{ .url = "imap://user%@example.com" },{ .url = "imap://user%00 at example.com" @@ -903,6 +907,7 @@ #endif "imap://user at host.example.com/", "imap://user at host.example.com:993/", + "imap://su%3auser at host.example.com/", "imap://user;AUTH=PLAIN at host.example.com/", "imap://user;AUTH=PLAIN at host.example.com/INBOX", "imap://user;AUTH=PLAIN at host.example.com/INBOX/;UID=5", From dovecot at dovecot.org Tue Sep 17 21:58:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 21:58:06 +0300 Subject: dovecot-2.2: lib-http: Added (non-default) support for parsing u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ec9eab188dd8 changeset: 16761:ec9eab188dd8 user: Stephan Bosch date: Tue Sep 17 21:57:48 2013 +0300 description: lib-http: Added (non-default) support for parsing user:pasword from HTTP URL. diffstat: src/lib-http/http-url.c | 44 ++++++++++++++++++++++++++++++++------------ src/lib-http/http-url.h | 10 ++++++++-- src/lib-http/test-http-url.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 14 deletions(-) diffs (156 lines): diff -r 6be5d8d8af2d -r ec9eab188dd8 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Tue Sep 17 21:57:14 2013 +0300 +++ b/src/lib-http/http-url.c Tue Sep 17 21:57:48 2013 +0300 @@ -32,25 +32,41 @@ struct uri_parser *parser = &url_parser->parser; struct http_url *url = url_parser->url; struct uri_authority auth; + const char *user = NULL, *password = NULL; int ret; if ((ret = uri_parse_authority(parser, &auth)) < 0) return FALSE; if (ret > 0) { if (auth.enc_userinfo != NULL) { - /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20 + const char *p; - Section 2.8.1: + if ((url_parser->flags & HTTP_URL_ALLOW_USERINFO_PART) == 0) { + /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20 - {...} Senders MUST NOT include a userinfo subcomponent (and its "@" - delimiter) when transmitting an "http" URI in a message. Recipients - of HTTP messages that contain a URI reference SHOULD parse for the - existence of userinfo and treat its presence as an error, likely - indicating that the deprecated subcomponent is being used to - obscure the authority for the sake of phishing attacks. - */ - parser->error = "HTTP URL does not allow `userinfo@' part"; - return FALSE; + Section 2.8.1: + + {...} Senders MUST NOT include a userinfo subcomponent (and its "@" + delimiter) when transmitting an "http" URI in a message. Recipients + of HTTP messages that contain a URI reference SHOULD parse for the + existence of userinfo and treat its presence as an error, likely + indicating that the deprecated subcomponent is being used to + obscure the authority for the sake of phishing attacks. + */ + parser->error = "HTTP URL does not allow `userinfo@' part"; + return FALSE; + } + + p = strchr(auth.enc_userinfo, ':'); + if (p == NULL) { + if (!uri_data_decode(parser, auth.enc_userinfo, NULL, &user)) + return FALSE; + } else { + if (!uri_data_decode(parser, auth.enc_userinfo, p, &user)) + return FALSE; + if (!uri_data_decode(parser, p+1, NULL, &password)) + return FALSE; + } } } if (url != NULL) { @@ -59,6 +75,8 @@ url->have_host_ip = auth.have_host_ip; url->port = auth.port; url->have_port = auth.have_port; + url->user = p_strdup(parser->pool, user); + url->password = p_strdup(parser->pool, password); } return TRUE; } @@ -210,12 +228,14 @@ parser->error = "Relative HTTP URL not allowed"; return FALSE; } else if (!have_authority && url != NULL) { - url->host_name = p_strdup(parser->pool, base->host_name); + url->host_name = p_strdup_empty(parser->pool, base->host_name); url->host_ip = base->host_ip; url->have_host_ip = base->have_host_ip; url->port = base->port; url->have_port = base->have_port; url->have_ssl = base->have_ssl; + url->user = p_strdup_empty(parser->pool, base->user); + url->password = p_strdup_empty(parser->pool, base->password); } url_parser->relative = TRUE; diff -r 6be5d8d8af2d -r ec9eab188dd8 src/lib-http/http-url.h --- a/src/lib-http/http-url.h Tue Sep 17 21:57:14 2013 +0300 +++ b/src/lib-http/http-url.h Tue Sep 17 21:57:48 2013 +0300 @@ -11,6 +11,10 @@ struct ip_addr host_ip; in_port_t port; + /* userinfo (not parsed by default) */ + const char *user; + const char *password; + /* path */ const char *path; @@ -33,8 +37,10 @@ /* Scheme part 'http:' is already parsed externally. This implies that this is an absolute HTTP URL. */ HTTP_URL_PARSE_SCHEME_EXTERNAL = 0x01, - /* Allow '#fragment' part in URL */ - HTTP_URL_ALLOW_FRAGMENT_PART = 0x02 + /* Allow '#fragment' part in HTTP URL */ + HTTP_URL_ALLOW_FRAGMENT_PART = 0x02, + /* Allow 'user:password@' part in HTTP URL */ + HTTP_URL_ALLOW_USERINFO_PART = 0x04 }; int http_url_parse(const char *url, struct http_url *base, diff -r 6be5d8d8af2d -r ec9eab188dd8 src/lib-http/test-http-url.c --- a/src/lib-http/test-http-url.c Tue Sep 17 21:57:14 2013 +0300 +++ b/src/lib-http/test-http-url.c Tue Sep 17 21:57:48 2013 +0300 @@ -48,6 +48,23 @@ .port = 8080, .have_port = TRUE } #endif },{ + .url = "http://user at api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host_name = "api.dovecot.org", .user = "user" } + },{ + .url = "http://userid:secret at api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host_name = "api.dovecot.org", + .user = "userid", .password = "secret" } + },{ + .url = "http://su%3auserid:secret at api.dovecot.org", + .flags = HTTP_URL_ALLOW_USERINFO_PART, + .url_parsed = { + .host_name = "api.dovecot.org", + .user = "su:userid", .password = "secret" } + },{ .url = "http://www.example.com/" "?question=What%20are%20you%20doing%3f&answer=Nothing.", .url_parsed = { @@ -296,6 +313,20 @@ test_out("url->host_ip = (valid)", urlp->have_host_ip == urlt->have_host_ip); } + if (urlp->user == NULL || urlt->user == NULL) { + test_out(t_strdup_printf("url->user = %s", urlp->user), + urlp->user == urlt->user); + } else { + test_out(t_strdup_printf("url->user = %s", urlp->user), + strcmp(urlp->user, urlt->user) == 0); + } + if (urlp->password == NULL || urlt->password == NULL) { + test_out(t_strdup_printf("url->password = %s", urlp->password), + urlp->password == urlt->password); + } else { + test_out(t_strdup_printf("url->password = %s", urlp->password), + strcmp(urlp->password, urlt->password) == 0); + } if (urlp->path == NULL || urlt->path == NULL) { test_out(t_strdup_printf("url->path = %s", urlp->path), urlp->path == urlt->path); From dovecot at dovecot.org Tue Sep 17 23:12:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 23:12:17 +0300 Subject: dovecot-2.2: dsync: Improved debug/error logging for changes dur... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/393c389b1540 changeset: 16762:393c389b1540 user: Timo Sirainen date: Tue Sep 17 23:12:03 2013 +0300 description: dsync: Improved debug/error logging for changes during sync. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 27 +++++++++++++++++++++- src/doveadm/dsync/dsync-brain-mailbox.c | 25 +++++++++++++++++---- src/doveadm/dsync/dsync-brain-private.h | 2 +- src/doveadm/dsync/dsync-mailbox-import.c | 12 ++++++++- 4 files changed, 56 insertions(+), 10 deletions(-) diffs (203 lines): diff -r ec9eab188dd8 -r 393c389b1540 src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Tue Sep 17 21:57:48 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Tue Sep 17 23:12:03 2013 +0300 @@ -111,8 +111,19 @@ case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: /* make sure we're deleting the correct mailbox */ ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid, - &box); - if (ret <= 0) { + &box, &errstr); + if (ret < 0) { + i_error("Mailbox sync: Couldn't allocate mailbox GUID %s: %s", + guid_128_to_string(change->mailbox_guid), errstr); + return -1; + } + if (ret == 0) { + if (brain->debug) { + i_debug("brain %c: Mailbox GUID %s sync: " + "Deletion conflict: %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), errstr); + } brain->changes_during_sync = TRUE; return ret; } @@ -126,6 +137,12 @@ errstr = mailbox_list_get_last_error(change->ns->list, &error); if (error == MAIL_ERROR_NOTFOUND || error == MAIL_ERROR_EXISTS) { + if (brain->debug) { + i_debug("brain %c: Mailbox %s sync: " + "mailbox_list_delete_dir conflict: %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), errstr); + } brain->changes_during_sync = TRUE; return 0; } else { @@ -175,6 +192,12 @@ error == MAIL_ERROR_NOTFOUND) { /* mailbox was already created or was already deleted. let the next sync figure out what to do */ + if (brain->debug) { + i_debug("brain %c: Mailbox %s sync: %s conflict: %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), + func_name, errstr); + } brain->changes_during_sync = TRUE; ret = 0; } else { diff -r ec9eab188dd8 -r 393c389b1540 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Tue Sep 17 21:57:48 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Tue Sep 17 23:12:03 2013 +0300 @@ -15,20 +15,25 @@ static int ns_mailbox_try_alloc(struct mail_namespace *ns, const guid_128_t guid, - struct mailbox **box_r) + struct mailbox **box_r, const char **error_r) { struct mailbox *box; enum mailbox_existence existence; + enum mail_error err; int ret; box = mailbox_alloc_guid(ns->list, guid, 0); ret = mailbox_exists(box, FALSE, &existence); if (ret < 0) { + *error_r = mailbox_get_last_error(box, &err); mailbox_free(&box); return -1; } if (existence != MAILBOX_EXISTENCE_SELECT) { mailbox_free(&box); + *error_r = existence == MAILBOX_EXISTENCE_NONE ? + "Mailbox was already deleted" : + "Mailbox is no longer selectable"; return 0; } *box_r = box; @@ -36,14 +41,14 @@ } int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, - struct mailbox **box_r) + struct mailbox **box_r, const char **error_r) { struct mail_namespace *ns; int ret; *box_r = NULL; if (brain->sync_ns != NULL) { - ret = ns_mailbox_try_alloc(brain->sync_ns, guid, box_r); + ret = ns_mailbox_try_alloc(brain->sync_ns, guid, box_r, error_r); if (ret < 0) brain->failed = TRUE; return ret; @@ -52,7 +57,7 @@ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { if (!dsync_brain_want_namespace(brain, ns)) continue; - if ((ret = ns_mailbox_try_alloc(ns, guid, box_r)) != 0) { + if ((ret = ns_mailbox_try_alloc(ns, guid, box_r, error_r)) != 0) { if (ret < 0) brain->failed = TRUE; return ret; @@ -596,6 +601,7 @@ const struct dsync_mailbox *dsync_box; struct dsync_mailbox local_dsync_box; struct mailbox *box; + const char *error; int ret; i_assert(!brain->master_brain); @@ -608,13 +614,22 @@ return TRUE; } - if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, &box) < 0) { + if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, + &box, &error) < 0) { + i_error("Couldn't allocate mailbox GUID %s: %s", + guid_128_to_string(dsync_box->mailbox_guid), error); i_assert(brain->failed); return TRUE; } if (box == NULL) { /* mailbox was probably deleted/renamed during sync */ //FIXME: verify this from log, and if not log an error. + if (brain->debug) { + i_debug("brain %c: Mailbox GUID %s sync: " + "Mailbox was lost during sync", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } brain->changes_during_sync = TRUE; dsync_brain_slave_send_mailbox_lost(brain, dsync_box); return TRUE; diff -r ec9eab188dd8 -r 393c389b1540 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Tue Sep 17 21:57:48 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Tue Sep 17 23:12:03 2013 +0300 @@ -113,7 +113,7 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, - struct mailbox **box_r); + struct mailbox **box_r, const char **error_r); void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_box, diff -r ec9eab188dd8 -r 393c389b1540 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Tue Sep 17 21:57:48 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Sep 17 23:12:03 2013 +0300 @@ -2020,9 +2020,10 @@ } static int -reassign_uids_in_seq_range(struct mailbox *box, +reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer, const ARRAY_TYPE(seq_range) *unwanted_uids) { + struct mailbox *box = importer->box; const enum mailbox_transaction_flags trans_flags = MAILBOX_TRANSACTION_FLAG_EXTERNAL | MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; @@ -2032,6 +2033,7 @@ struct mail_search_context *search_ctx; struct mail_save_context *save_ctx; struct mail *mail; + unsigned int renumber_count = 0; int ret = 1; if (array_count(unwanted_uids) == 0) @@ -2058,6 +2060,7 @@ } else if (ret > 0) { ret = 0; } + renumber_count++; } if (mailbox_search_deinit(&search_ctx) < 0) { i_error("Mailbox %s: mail search failed: %s", @@ -2072,6 +2075,11 @@ mailbox_get_last_error(box, NULL)); ret = -1; } + if (ret == 0 && importer->debug) { + i_debug("Mailbox %s: Renumbered %u of %u unwanted UIDs", + mailbox_get_vname(box), + renumber_count, array_count(unwanted_uids)); + } return ret; } @@ -2119,7 +2127,7 @@ seq_range_array_remove(&unwanted_uids, saved_uids[i]); } - ret = reassign_uids_in_seq_range(importer->box, &unwanted_uids); + ret = reassign_uids_in_seq_range(importer, &unwanted_uids); if (ret == 0) { *changes_during_sync_r = TRUE; /* conflicting changes during sync, revert our last-common-uid From dovecot at dovecot.org Tue Sep 17 23:33:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 17 Sep 2013 23:33:06 +0300 Subject: dovecot-2.2: lib-storage: Added a separate prefix for server met... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/194383b51917 changeset: 16763:194383b51917 user: Timo Sirainen date: Tue Sep 17 23:32:57 2013 +0300 description: lib-storage: Added a separate prefix for server metadata that aren't deleted with INBOX. diffstat: src/lib-storage/index/index-storage.c | 6 ++++++ src/lib-storage/mail-storage.h | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 393c389b1540 -r 194383b51917 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Tue Sep 17 23:12:03 2013 +0300 +++ b/src/lib-storage/index/index-storage.c Tue Sep 17 23:32:57 2013 +0300 @@ -614,9 +614,15 @@ struct mailbox_attribute_iter *iter; const char *key; int ret = 0; + bool inbox = t->box->inbox_any; iter = mailbox_attribute_iter_init(t->box, type, ""); while ((key = mailbox_attribute_iter_next(iter)) != NULL) { + if (inbox && + strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, + strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER)) == 0) + continue; + if (mailbox_attribute_unset(t, type, key) < 0) ret = -1; } diff -r 393c389b1540 -r 194383b51917 src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Tue Sep 17 23:12:03 2013 +0300 +++ b/src/lib-storage/mail-storage.h Tue Sep 17 23:32:57 2013 +0300 @@ -209,7 +209,12 @@ #define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT "vendor/vendor.dovecot/" /* Prefix used for attributes reserved for Dovecot's internal use. Normal users cannot access these in any way. */ -#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT "vendor/vendor.dovecot/pvt/" +#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT \ + MAILBOX_ATTRIBUTE_PREFIX_DOVECOT"pvt/" +/* Prefix used for server attributes in INBOX. INBOX deletion won't delete + any attributes under this prefix. */ +#define MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER \ + MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT"server/" enum mail_attribute_type { MAIL_ATTRIBUTE_TYPE_PRIVATE, From dovecot at dovecot.org Wed Sep 18 00:00:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Sep 2013 00:00:17 +0300 Subject: dovecot-2.2: dsync: Don't attempt to access mailbox contents wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7be7ee98538 changeset: 16764:a7be7ee98538 user: Timo Sirainen date: Wed Sep 18 00:00:05 2013 +0300 description: dsync: Don't attempt to access mailbox contents with -g "" or -m "" parameter. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 194383b51917 -r a7be7ee98538 src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Tue Sep 17 23:32:57 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Wed Sep 18 00:00:05 2013 +0300 @@ -28,6 +28,11 @@ return -1; } } + if (brain->no_mail_sync) { + /* trust that create worked, we can't actually open it + and verify. */ + return 0; + } /* sync the mailbox so we can look up its latest status */ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { i_error("Can't sync mailbox %s: %s", From pigeonhole at rename-it.nl Wed Sep 18 00:57:44 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 17 Sep 2013 23:57:44 +0200 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Small initializati... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/4a11a588b55c changeset: 1803:4a11a588b55c user: Stephan Bosch date: Tue Sep 17 23:55:59 2013 +0200 description: doveadm sieve plugin: Small initialization fix. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b1db52d0c0b3 -r 4a11a588b55c src/plugins/doveadm-sieve/doveadm-sieve-plugin.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Mon Sep 16 01:15:55 2013 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Tue Sep 17 23:55:59 2013 +0200 @@ -80,7 +80,7 @@ if (suser != NULL) { *svstorage_r = suser->sieve_storage; - return 0; + return suser->sieve_storage != NULL ? 1 : 0; } /* Delayed initialization of sieve storage until it's actually needed */ From pigeonhole at rename-it.nl Wed Sep 18 00:57:44 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 17 Sep 2013 23:57:44 +0200 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Changed root attri... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/34a5db66fc52 changeset: 1804:34a5db66fc52 user: Stephan Bosch date: Tue Sep 17 23:57:05 2013 +0200 description: doveadm sieve plugin: Changed root attribute for Sieve. diffstat: src/lib-sievestorage/sieve-storage.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4a11a588b55c -r 34a5db66fc52 src/lib-sievestorage/sieve-storage.h --- a/src/lib-sievestorage/sieve-storage.h Tue Sep 17 23:55:59 2013 +0200 +++ b/src/lib-sievestorage/sieve-storage.h Tue Sep 17 23:57:05 2013 +0200 @@ -11,7 +11,7 @@ #include "sieve.h" #define MAILBOX_ATTRIBUTE_PREFIX_SIEVE \ - MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT"sieve/" + MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER"sieve/" #define MAILBOX_ATTRIBUTE_PREFIX_SIEVE_FILES \ MAILBOX_ATTRIBUTE_PREFIX_SIEVE"files/" #define MAILBOX_ATTRIBUTE_SIEVE_DEFAULT \ From dovecot at dovecot.org Wed Sep 18 23:24:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Sep 2013 23:24:48 +0300 Subject: dovecot-2.2: lib-http: Fixed handling of limits=NULL parameter f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/34d3a9a883fa changeset: 16765:34d3a9a883fa user: Stephan Bosch date: Wed Sep 18 23:24:02 2013 +0300 description: lib-http: Fixed handling of limits=NULL parameter for http_request_parser_init. Got messed up in patch queue. diffstat: src/lib-http/http-request-parser.c | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diffs (25 lines): diff -r a7be7ee98538 -r 34d3a9a883fa src/lib-http/http-request-parser.c --- a/src/lib-http/http-request-parser.c Wed Sep 18 00:00:05 2013 +0300 +++ b/src/lib-http/http-request-parser.c Wed Sep 18 23:24:02 2013 +0300 @@ -42,15 +42,16 @@ { struct http_request_parser *parser; struct http_header_limits hdr_limits; - uoff_t max_payload_size = limits->max_payload_size; + uoff_t max_payload_size; parser = i_new(struct http_request_parser, 1); - parser->max_target_length = limits->max_target_length; - - if (limits != NULL) + if (limits != NULL) { hdr_limits = limits->header; - else + max_payload_size = limits->max_payload_size; + } else { memset(&hdr_limits, 0, sizeof(hdr_limits)); + max_payload_size = 0; + } /* substitute default limits */ if (parser->max_target_length == 0) From dovecot at dovecot.org Wed Sep 18 23:24:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Sep 2013 23:24:48 +0300 Subject: dovecot-2.2: lib-http: http-url: Fixed return of proper error me... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8442a0dc44b7 changeset: 16766:8442a0dc44b7 user: Stephan Bosch date: Wed Sep 18 23:24:11 2013 +0300 description: lib-http: http-url: Fixed return of proper error message in case of a failure to parse request target Host header. Assigned the error to the parser object rather than returning it directly. diffstat: src/lib-http/http-url.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 34d3a9a883fa -r 8442a0dc44b7 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Wed Sep 18 23:24:02 2013 +0300 +++ b/src/lib-http/http-url.c Wed Sep 18 23:24:11 2013 +0300 @@ -370,12 +370,12 @@ uri_parser_init(parser, pool, host_header); if (uri_parse_authority(parser, &host) <= 0) { - parser->error = t_strdup_printf("Invalid Host header: %s", parser->error); + *error_r = t_strdup_printf("Invalid Host header: %s", parser->error); return -1; } if (parser->cur != parser->end || host.enc_userinfo != NULL) { - parser->error = "Invalid Host header: Contains invalid character"; + *error_r = "Invalid Host header: Contains invalid character"; return -1; } From dovecot at dovecot.org Wed Sep 18 23:24:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Sep 2013 23:24:48 +0300 Subject: dovecot-2.2: uri-util: Improved authority 'host' parse error. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9bf25c18ad33 changeset: 16767:9bf25c18ad33 user: Stephan Bosch date: Wed Sep 18 23:24:22 2013 +0300 description: uri-util: Improved authority 'host' parse error. diffstat: src/lib/uri-util.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 8442a0dc44b7 -r 9bf25c18ad33 src/lib/uri-util.c --- a/src/lib/uri-util.c Wed Sep 18 23:24:11 2013 +0300 +++ b/src/lib/uri-util.c Wed Sep 18 23:24:22 2013 +0300 @@ -543,7 +543,10 @@ /* host */ if ((ret = uri_parse_host(parser, auth)) <= 0) { if (ret == 0) { - parser->error = "Missing 'host' component"; + if (p == parser->end || *p == ':' || *p == '/') + parser->error = "Missing 'host' component"; + else + parser->error = "Invalid 'host' component"; return -1; } return ret; From dovecot at dovecot.org Wed Sep 18 23:24:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 18 Sep 2013 23:24:48 +0300 Subject: dovecot-2.2: lib-http: Created tests for http_request_parser. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b92118278b06 changeset: 16768:b92118278b06 user: Stephan Bosch date: Wed Sep 18 23:24:30 2013 +0300 description: lib-http: Created tests for http_request_parser. diffstat: src/lib-http/Makefile.am | 17 +- src/lib-http/test-http-request-parser.c | 271 ++++++++++++++++++++++++++++++++ 2 files changed, 287 insertions(+), 1 deletions(-) diffs (truncated from 309 to 300 lines): diff -r 9bf25c18ad33 -r b92118278b06 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Wed Sep 18 23:24:22 2013 +0300 +++ b/src/lib-http/Makefile.am Wed Sep 18 23:24:30 2013 +0300 @@ -47,7 +47,8 @@ test-http-url \ test-http-header-parser \ test-http-transfer \ - test-http-response-parser + test-http-response-parser \ + test-http-request-parser test_nocheck_programs = \ test-http-client \ @@ -97,6 +98,20 @@ $(test_libs) test_http_response_parser_DEPENDENCIES = $(test_deps) +test_http_request_parser_SOURCES = test-http-request-parser.c +test_http_request_parser_LDADD = \ + http-date.lo \ + http-parser.lo \ + http-url.lo \ + http-header.lo \ + http-header-parser.lo \ + http-transfer-chunked.lo \ + http-message-parser.lo \ + http-request-parser.lo \ + $(test_libs) +test_http_request_parser_DEPENDENCIES = $(test_deps) + + test_http_client_SOURCES = test-http-client.c test_http_client_LDFLAGS = -export-dynamic test_http_client_LDADD = \ diff -r 9bf25c18ad33 -r b92118278b06 src/lib-http/test-http-request-parser.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/test-http-request-parser.c Wed Sep 18 23:24:30 2013 +0300 @@ -0,0 +1,271 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "buffer.h" +#include "str.h" +#include "str-sanitize.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "http-url.h" +#include "http-request-parser.h" + +#include + +struct http_request_parse_test { + const char *request; + const char *method; + const char *target_raw; + struct http_request_target target; + unsigned char version_major; + unsigned char version_minor; + uoff_t content_length; + const char *payload; +}; + +/* Valid header tests */ + +static const struct http_request_parse_test +valid_request_parse_tests[] = { + { .request = + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .method = "GET", + .target_raw = "/", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN + }, + .version_major = 1, .version_minor = 1, + },{ .request = + "OPTIONS * HTTP/1.0\r\n" + "Host: example.com\r\n" + "\r\n", + .method = "OPTIONS", + .target_raw = "*", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK + }, + .version_major = 1, .version_minor = 0, + },{ .request = + "CONNECT example.com:443 HTTP/1.2\r\n" + "Host: example.com:443\r\n" + "\r\n", + .method = "CONNECT", + .target_raw = "example.com:443", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY + }, + .version_major = 1, .version_minor = 2, + },{ .request = + "GET https://www.example.com:443 HTTP/1.1\r\n" + "Host: www.example.com:80\r\n" + "\r\n", + .method = "GET", + .target_raw = "https://www.example.com:443", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE + }, + .version_major = 1, .version_minor = 1, + },{ .request = + "POST http://api.example.com:8080/commit?user=dirk HTTP/1.1\r\n" + "Host: api.example.com:8080\r\n" + "Content-Length: 10\r\n" + "\r\n" + "Content!\r\n", + .method = "POST", + .target_raw = "http://api.example.com:8080/commit?user=dirk", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + }, + .version_major = 1, .version_minor = 1, + .payload = "Content!\r\n" + } +}; + +unsigned int valid_request_parse_test_count = + N_ELEMENTS(valid_request_parse_tests); + +static const char * +_request_target_format(enum http_request_target_format target_format) +{ + switch (target_format) { + case HTTP_REQUEST_TARGET_FORMAT_ORIGIN: + return "origin"; + case HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE: + return "absolute"; + case HTTP_REQUEST_TARGET_FORMAT_AUTHORITY: + return "authority"; + case HTTP_REQUEST_TARGET_FORMAT_ASTERISK: + return "asterisk"; + } + return t_strdup_printf("<>", target_format); +} + +static void test_http_request_parse_valid(void) +{ + unsigned int i; + buffer_t *payload_buffer = buffer_create_dynamic(default_pool, 1024); + + for (i = 0; i < valid_request_parse_test_count; i++) T_BEGIN { + struct istream *input; + struct ostream *output; + const struct http_request_parse_test *test; + struct http_request_parser *parser; + struct http_request request; + enum http_request_parse_error error_code; + const char *request_text, *payload, *error; + unsigned int pos, request_text_len; + int ret = 0; + + test = &valid_request_parse_tests[i]; + request_text = test->request; + request_text_len = strlen(request_text); + input = test_istream_create_data(request_text, request_text_len); + parser = http_request_parser_init(input, NULL); + + test_begin(t_strdup_printf("http request valid [%d]", i)); + + payload = NULL; + for (pos = 0; pos <= request_text_len && ret == 0; pos++) { + test_istream_set_size(input, pos); + ret = http_request_parse_next + (parser, FALSE, &request, &error_code, &error); + } + test_istream_set_size(input, request_text_len); + while (ret > 0) { + if (request.payload != NULL) { + buffer_set_used_size(payload_buffer, 0); + output = o_stream_create_buffer(payload_buffer); + test_out("payload receive", + o_stream_send_istream(output, request.payload)); + o_stream_destroy(&output); + payload = str_c(payload_buffer); + } else { + payload = NULL; + } + ret = http_request_parse_next + (parser, FALSE, &request, &error_code, &error); + } + + test_out_reason("parse success", ret == 0, error); + + if (ret == 0) { + /* verify last request only */ + if (request.method == NULL || test->method == NULL) { + test_out(t_strdup_printf("request->method = %s", test->method), + request.method == test->method); + } else { + test_out(t_strdup_printf("request->method = %s", test->method), + strcmp(request.method, test->method) == 0); + } + if (request.target_raw == NULL || test->target_raw == NULL) { + test_out(t_strdup_printf("request->target = %s", test->target_raw), + request.target_raw == test->target_raw); + } else { + test_out(t_strdup_printf("request->target = %s", test->target_raw), + strcmp(request.target_raw, test->target_raw) == 0); + } + test_out(t_strdup_printf("request->target_format = %s", + _request_target_format(test->target.format)), + request.target.format == test->target.format); + test_out(t_strdup_printf("request->version = %d.%d", + test->version_major, test->version_minor), + request.version_major == test->version_major && + request.version_minor == test->version_minor); + if (payload == NULL || test->payload == NULL) { + test_out(t_strdup_printf("request->payload = %s", + str_sanitize(payload, 80)), + payload == test->payload); + } else { + test_out(t_strdup_printf("request->payload = %s", + str_sanitize(payload, 80)), + strcmp(payload, test->payload) == 0); + } + } + test_end(); + http_request_parser_deinit(&parser); + } T_END; + + buffer_free(&payload_buffer); +} + +static const char *invalid_request_parse_tests[] = { + "GET: / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + "GET % HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + "GET /frop\" HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + "GET / HTCPCP/1.0\r\n" + "Host: example.com\r\n" + "\r\n", + "GET / HTTP/1.0.1\r\n" + "Host: example.com\r\n" + "\r\n", + "GET / HTTP/1.1\r\n" + "Host: \"example.com\r\n" + "\r\n", +}; + +static unsigned char invalid_request_with_nuls[] = + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Null: text\0server\r\n" + "\r\n"; + +unsigned int invalid_request_parse_test_count = + N_ELEMENTS(invalid_request_parse_tests); + +static void test_http_request_parse_invalid(void) +{ + struct http_request_parser *parser; + struct http_request request; + enum http_request_parse_error error_code; + const char *request_text, *error; + struct istream *input; + int ret; + unsigned int i; + + for (i = 0; i < invalid_request_parse_test_count; i++) T_BEGIN { + const char *test; + + test = invalid_request_parse_tests[i]; + request_text = test; + input = i_stream_create_from_data(request_text, strlen(request_text)); + parser = http_request_parser_init(input, NULL); + + test_begin(t_strdup_printf("http request invalid [%d]", i)); + + while ((ret=http_request_parse_next + (parser, FALSE, &request, &error_code, &error)) > 0); + + test_out_reason("parse failure", ret < 0, error); + test_end(); + http_request_parser_deinit(&parser); + } T_END; + + /* parse failure guarantees http_request_header.size equals + strlen(http_request_header.value) */ + test_begin("http request with NULs"); + input = i_stream_create_from_data(invalid_request_with_nuls, + sizeof(invalid_request_with_nuls)-1); + parser = http_request_parser_init(input, 0); + while ((ret=http_request_parse_next + (parser, FALSE, &request, &error_code, &error)) > 0); + test_assert(ret < 0); + test_end(); +} + From dovecot at dovecot.org Thu Sep 19 22:21:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Sep 2013 22:21:15 +0300 Subject: dovecot-2.2: imapc: Abort pending commands before any deinitiali... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/827ebbf9da5f changeset: 16769:827ebbf9da5f user: Timo Sirainen date: Thu Sep 19 22:20:56 2013 +0300 description: imapc: Abort pending commands before any deinitialization to avoid crashes. diffstat: src/lib-imap-client/imapc-client.c | 24 +++++++++++++++++------- src/lib-imap-client/imapc-client.h | 1 + src/lib-storage/index/imapc/imapc-list.c | 6 +++++- src/lib-storage/index/imapc/imapc-storage.c | 4 ++++ 4 files changed, 27 insertions(+), 8 deletions(-) diffs (84 lines): diff -r b92118278b06 -r 827ebbf9da5f src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Wed Sep 18 23:24:30 2013 +0300 +++ b/src/lib-imap-client/imapc-client.c Thu Sep 19 22:20:56 2013 +0300 @@ -110,17 +110,27 @@ pool_unref(&client->pool); } +void imapc_client_disconnect(struct imapc_client *client) +{ + struct imapc_client_connection *const *conns, *conn; + unsigned int i, count; + + conns = array_get(&client->conns, &count); + for (i = count; i > 0; i--) { + conn = conns[i-1]; + array_delete(&client->conns, i-1, 1); + + i_assert(imapc_connection_get_mailbox(conn->conn) == NULL); + imapc_connection_deinit(&conn->conn); + i_free(conn); + } +} + void imapc_client_deinit(struct imapc_client **_client) { struct imapc_client *client = *_client; - struct imapc_client_connection **connp; - array_foreach_modifiable(&client->conns, connp) { - i_assert(imapc_connection_get_mailbox((*connp)->conn) == NULL); - imapc_connection_deinit(&(*connp)->conn); - i_free(*connp); - } - array_clear(&client->conns); + imapc_client_disconnect(client); imapc_client_unref(_client); } diff -r b92118278b06 -r 827ebbf9da5f src/lib-imap-client/imapc-client.h --- a/src/lib-imap-client/imapc-client.h Wed Sep 18 23:24:30 2013 +0300 +++ b/src/lib-imap-client/imapc-client.h Thu Sep 19 22:20:56 2013 +0300 @@ -134,6 +134,7 @@ struct imapc_client * imapc_client_init(const struct imapc_client_settings *set); +void imapc_client_disconnect(struct imapc_client *client); void imapc_client_deinit(struct imapc_client **client); /* Explicitly login to server (also done automatically). */ diff -r b92118278b06 -r 827ebbf9da5f src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Wed Sep 18 23:24:30 2013 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Thu Sep 19 22:20:56 2013 +0300 @@ -98,12 +98,16 @@ { struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + /* make sure all pending commands are aborted before anything is + deinitialized */ + imapc_client_disconnect(list->client->client); + + imapc_storage_client_unref(&list->client); if (list->index_list != NULL) mailbox_list_destroy(&list->index_list); mailbox_tree_deinit(&list->mailboxes); if (list->tmp_subscriptions != NULL) mailbox_tree_deinit(&list->tmp_subscriptions); - imapc_storage_client_unref(&list->client); pool_unref(&list->list.pool); } diff -r b92118278b06 -r 827ebbf9da5f src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Wed Sep 18 23:24:30 2013 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Thu Sep 19 22:20:56 2013 +0300 @@ -295,6 +295,10 @@ { struct imapc_storage *storage = (struct imapc_storage *)_storage; + /* make sure all pending commands are aborted before anything is + deinitialized */ + imapc_client_disconnect(storage->client->client); + imapc_storage_client_unref(&storage->client); index_storage_destroy(_storage); } From dovecot at dovecot.org Thu Sep 19 22:44:00 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Sep 2013 22:44:00 +0300 Subject: dovecot-2.2: auth: Fixed non-auth passdb lookup when password ha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ed1f5b4f38be changeset: 16770:ed1f5b4f38be user: Timo Sirainen date: Thu Sep 19 22:43:45 2013 +0300 description: auth: Fixed non-auth passdb lookup when password had "." suffix. diffstat: src/auth/passdb.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 827ebbf9da5f -r ed1f5b4f38be src/auth/passdb.c --- a/src/auth/passdb.c Thu Sep 19 22:20:56 2013 +0300 +++ b/src/auth/passdb.c Thu Sep 19 22:43:45 2013 +0300 @@ -92,7 +92,7 @@ /* anything goes. change the credentials_scheme to what we actually got, so blocking passdbs work. */ auth_request->credentials_scheme = - p_strdup(auth_request->pool, input_scheme); + p_strdup(auth_request->pool, t_strcut(input_scheme, '.')); return TRUE; } From dovecot at dovecot.org Thu Sep 19 22:44:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 19 Sep 2013 22:44:28 +0300 Subject: dovecot-2.2: doveadm: Added "auth lookup" command to do a passdb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5a334879572b changeset: 16771:5a334879572b user: Timo Sirainen date: Thu Sep 19 22:44:20 2013 +0300 description: doveadm: Added "auth lookup" command to do a passdb lookup. diffstat: src/doveadm/doveadm-auth.c | 72 ++++++++++++++++++++++++++++++++++++++++----- 1 files changed, 64 insertions(+), 8 deletions(-) diffs (125 lines): diff -r ed1f5b4f38be -r 5a334879572b src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Thu Sep 19 22:43:45 2013 +0300 +++ b/src/doveadm/doveadm-auth.c Thu Sep 19 22:44:20 2013 +0300 @@ -31,8 +31,9 @@ static int cmd_user_input(const char *auth_socket_path, const struct authtest_input *input, - const char *show_field) + const char *show_field, bool userdb) { + const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup"; struct auth_master_connection *conn; pool_t pool; const char *username, *const *fields, *p; @@ -46,18 +47,23 @@ pool = pool_alloconly_create("auth master lookup", 1024); conn = auth_master_init(auth_socket_path, 0); - ret = auth_master_user_lookup(conn, input->username, &input->info, - pool, &username, &fields); + if (userdb) { + ret = auth_master_user_lookup(conn, input->username, &input->info, + pool, &username, &fields); + } else { + ret = auth_master_pass_lookup(conn, input->username, &input->info, + pool, &fields); + } if (ret < 0) { if (fields[0] == NULL) - i_error("userdb lookup failed for %s", input->username); + i_error("%s failed for %s", lookup_name, input->username); else { - i_error("userdb lookup failed for %s: %s", + i_error("%s failed for %s: %s", lookup_name, input->username, fields[0]); } } else if (ret == 0) { fprintf(show_field == NULL ? stdout : stderr, - "userdb lookup: user %s doesn't exist\n", + "%s: user %s doesn't exist\n", lookup_name, input->username); } else if (show_field != NULL) { unsigned int show_field_len = strlen(show_field); @@ -68,7 +74,7 @@ printf("%s\n", *fields + show_field_len + 1); } } else { - printf("userdb: %s\n", input->username); + printf("%s: %s\n", userdb ? "userdb" : "passdb", input->username); for (; *fields; fields++) { p = strchr(*fields, '='); @@ -284,6 +290,54 @@ doveadm_exit_code = EX_NOPERM; } +static void cmd_auth_lookup(int argc, char *argv[]) +{ + const char *auth_socket_path = NULL; + struct authtest_input input; + const char *show_field = NULL; + bool first = TRUE; + int c, ret; + + memset(&input, 0, sizeof(input)); + input.info.service = "doveadm"; + + while ((c = getopt(argc, argv, "a:f:x:")) > 0) { + switch (c) { + case 'a': + auth_socket_path = optarg; + break; + case 'f': + show_field = optarg; + break; + case 'x': + auth_user_info_parse(&input.info, optarg); + break; + default: + auth_cmd_help(cmd_auth_lookup); + } + } + + if (optind == argc) + auth_cmd_help(cmd_auth_lookup); + + while ((input.username = argv[optind++]) != NULL) { + if (first) + first = FALSE; + else + putchar('\n'); + + ret = cmd_user_input(auth_socket_path, &input, show_field, FALSE); + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; + } + } +} + static void cmd_user_mail_input_field(const char *key, const char *value, const char *show_field) { @@ -435,7 +489,7 @@ ret = !userdb_only ? cmd_user_mail_input(storage_service, &input, show_field) : - cmd_user_input(auth_socket_path, &input, show_field); + cmd_user_input(auth_socket_path, &input, show_field, TRUE); switch (ret) { case -1: doveadm_exit_code = EX_TEMPFAIL; @@ -452,6 +506,8 @@ struct doveadm_cmd doveadm_cmd_auth[] = { { cmd_auth_test, "auth test", "[-a ] [-x ] []" }, + { cmd_auth_lookup, "auth lookup", + "[-a ] [-x ] [-f field] [...]" }, { cmd_auth_cache_flush, "auth cache flush", "[-a ] [ [...]]" }, { cmd_user, "user", From dovecot at dovecot.org Fri Sep 20 00:13:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:13:04 +0300 Subject: dovecot-2.2: iostream: Added ability to set/get error strings fo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e35be66003e2 changeset: 16772:e35be66003e2 user: Timo Sirainen date: Fri Sep 20 00:00:49 2013 +0300 description: iostream: Added ability to set/get error strings for streams. diffstat: src/lib/iostream-private.h | 7 +++++++ src/lib/iostream.c | 12 ++++++++++++ src/lib/istream.c | 18 ++++++++++++++++++ src/lib/istream.h | 2 ++ src/lib/ostream.c | 33 +++++++++++++++++++++++++++++---- src/lib/ostream.h | 3 +++ 6 files changed, 71 insertions(+), 4 deletions(-) diffs (187 lines): diff -r 5a334879572b -r e35be66003e2 src/lib/iostream-private.h --- a/src/lib/iostream-private.h Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/iostream-private.h Fri Sep 20 00:00:49 2013 +0300 @@ -11,6 +11,7 @@ struct iostream_private { int refcount; char *name; + char *error; void (*close)(struct iostream_private *streami, bool close_parent); void (*destroy)(struct iostream_private *stream); @@ -26,5 +27,11 @@ void io_stream_close(struct iostream_private *stream, bool close_parent); void io_stream_set_max_buffer_size(struct iostream_private *stream, size_t max_size); +/* Set a specific error for the stream. This shouldn't be used for regular + syscall errors where stream's errno is enough, since it's used by default. + The stream errno must always be set even if the error string is also set. + Setting this error replaces the previously set error. */ +void io_stream_set_error(struct iostream_private *stream, + const char *fmt, ...) ATTR_FORMAT(2, 3); #endif diff -r 5a334879572b -r e35be66003e2 src/lib/iostream.c --- a/src/lib/iostream.c Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/iostream.c Fri Sep 20 00:00:49 2013 +0300 @@ -46,6 +46,7 @@ array_free(&stream->destroy_callbacks); } + i_free(stream->error); i_free(stream->name); i_free(stream); } @@ -60,3 +61,14 @@ { stream->set_max_buffer_size(stream, max_size); } + +void io_stream_set_error(struct iostream_private *stream, + const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + i_free(stream->error); + stream->error = i_strdup_vprintf(fmt, args); + va_end(args); +} diff -r 5a334879572b -r e35be66003e2 src/lib/istream.c --- a/src/lib/istream.c Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/istream.c Fri Sep 20 00:00:49 2013 +0300 @@ -92,6 +92,24 @@ return _stream->fd; } +const char *i_stream_get_error(struct istream *stream) +{ + struct istream *s; + + /* we'll only return errors for streams that have stream_errno set. + we might be returning unintended error otherwise. */ + if (stream->stream_errno == 0) + return ""; + + for (s = stream; s != NULL; s = s->real_stream->parent) { + if (s->stream_errno == 0) + break; + if (s->real_stream->iostream.error != NULL) + return s->real_stream->iostream.error; + } + return strerror(stream->stream_errno); +} + void i_stream_close(struct istream *stream) { i_stream_close_full(stream, TRUE); diff -r 5a334879572b -r e35be66003e2 src/lib/istream.h --- a/src/lib/istream.h Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/istream.h Fri Sep 20 00:00:49 2013 +0300 @@ -65,6 +65,8 @@ /* Return file descriptor for stream, or -1 if none is available. */ int i_stream_get_fd(struct istream *stream); +/* Returns error string for the last error. */ +const char *i_stream_get_error(struct istream *stream); /* Mark the stream and all of its parent streams closed. Any reads after this will return -1. The data already read can still be used. */ diff -r 5a334879572b -r e35be66003e2 src/lib/ostream.c --- a/src/lib/ostream.c Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/ostream.c Fri Sep 20 00:00:49 2013 +0300 @@ -25,6 +25,24 @@ return stream->real_stream->fd; } +const char *o_stream_get_error(struct ostream *stream) +{ + struct ostream *s; + + /* we'll only return errors for streams that have stream_errno set. + we might be returning unintended error otherwise. */ + if (stream->stream_errno == 0) + return ""; + + for (s = stream; s != NULL; s = s->real_stream->parent) { + if (s->stream_errno == 0) + break; + if (s->real_stream->iostream.error != NULL) + return s->real_stream->iostream.error; + } + return strerror(stream->stream_errno); +} + static void o_stream_close_full(struct ostream *stream, bool close_parents) { io_stream_close(&stream->real_stream->iostream, close_parents); @@ -107,6 +125,12 @@ _stream->cork(_stream, FALSE); } +static void o_stream_clear_error(struct ostream *stream) +{ + stream->stream_errno = 0; + i_free_and_null(stream->real_stream->iostream.error); +} + int o_stream_flush(struct ostream *stream) { struct ostream_private *_stream = stream->real_stream; @@ -117,7 +141,7 @@ return -1; } - stream->stream_errno = 0; + o_stream_clear_error(stream); if (unlikely((ret = _stream->flush(_stream)) < 0)) { i_assert(stream->stream_errno != 0); stream->last_failed_errno = stream->stream_errno; @@ -160,7 +184,7 @@ return -1; } - stream->stream_errno = 0; + o_stream_clear_error(stream); if (unlikely(_stream->seek(_stream, offset) < 0)) { i_assert(stream->stream_errno != 0); stream->last_failed_errno = stream->stream_errno; @@ -194,7 +218,7 @@ return -1; } - stream->stream_errno = 0; + o_stream_clear_error(stream); for (i = 0, total_size = 0; i < iov_count; i++) total_size += iov[i].iov_len; if (total_size == 0) @@ -283,7 +307,7 @@ return -1; } - outstream->stream_errno = 0; + o_stream_clear_error(outstream); ret = _outstream->send_istream(_outstream, instream); if (unlikely(ret < 0)) { i_assert(outstream->stream_errno != 0); @@ -303,6 +327,7 @@ return -1; } + o_stream_clear_error(stream); ret = stream->real_stream->write_at(stream->real_stream, data, size, offset); if (unlikely(ret < 0)) { diff -r 5a334879572b -r e35be66003e2 src/lib/ostream.h --- a/src/lib/ostream.h Thu Sep 19 22:44:20 2013 +0300 +++ b/src/lib/ostream.h Fri Sep 20 00:00:49 2013 +0300 @@ -46,6 +46,9 @@ /* Return file descriptor for stream, or -1 if none is available. */ int o_stream_get_fd(struct ostream *stream); +/* Returns error string for the previous error (stream_errno, + not last_failed_errno). */ +const char *o_stream_get_error(struct ostream *stream); /* Close this stream (but not its parents) and unreference it. */ void o_stream_destroy(struct ostream **stream); From dovecot at dovecot.org Fri Sep 20 00:13:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:13:04 +0300 Subject: dovecot-2.2: iostreams: Set stream error string when it provides... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/76d5e3c8cec3 changeset: 16773:76d5e3c8cec3 user: Timo Sirainen date: Fri Sep 20 00:12:45 2013 +0300 description: iostreams: Set stream error string when it provides extra information. diffstat: src/lib-compression/istream-bzlib.c | 23 ++++++++--------- src/lib-compression/istream-zlib.c | 36 ++++++++++---------------- src/lib-fs/istream-metawrap.c | 2 + src/lib-mail/istream-dot.c | 2 + src/lib-mail/istream-qp-decoder.c | 11 +++++++- src/lib-ssl-iostream/iostream-openssl.c | 23 +++++++++++++++-- src/lib-ssl-iostream/iostream-openssl.h | 1 + src/lib-ssl-iostream/istream-openssl.c | 4 +++ src/lib-ssl-iostream/ostream-openssl.c | 4 +++ src/lib-storage/index/istream-mail.c | 4 ++- src/lib-storage/index/mbox/istream-raw-mbox.c | 19 +++++++++----- src/lib/istream-base64-decoder.c | 11 +++++++- src/lib/istream-concat.c | 10 +++++-- src/lib/istream-file.c | 13 ++++++--- src/lib/istream-hash.c | 3 +- src/lib/istream-jsonstr.c | 2 + src/lib/istream-mmap.c | 2 + src/lib/istream-sized.c | 19 ++++++++++--- src/lib/ostream-file.c | 10 ++++++- 19 files changed, 138 insertions(+), 61 deletions(-) diffs (truncated from 634 to 300 lines): diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-compression/istream-bzlib.c --- a/src/lib-compression/istream-bzlib.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-compression/istream-bzlib.c Fri Sep 20 00:12:45 2013 +0300 @@ -38,10 +38,13 @@ static void bzlib_read_error(struct bzlib_istream *zstream, const char *error) { - i_error("bzlib.read(%s): %s at %"PRIuUOFF_T, - i_stream_get_name(&zstream->istream.istream), error, - zstream->istream.abs_start_offset + - zstream->istream.istream.v_offset); + io_stream_set_error(&zstream->istream.iostream, + "bzlib.read(%s): %s at %"PRIuUOFF_T, + i_stream_get_name(&zstream->istream.istream), error, + zstream->istream.abs_start_offset + + zstream->istream.istream.v_offset); + if (zstream->log_errors) + i_error("%s", zstream->istream.iostream.error); } static ssize_t i_stream_bzlib_read(struct istream_private *stream) @@ -104,8 +107,7 @@ stream->parent->stream_errno; } else { i_assert(stream->parent->eof); - if (zstream->log_errors) - bzlib_read_error(zstream, "unexpected EOF"); + bzlib_read_error(zstream, "unexpected EOF"); stream->istream.stream_errno = EINVAL; } return -1; @@ -135,15 +137,12 @@ case BZ_PARAM_ERROR: i_unreached(); case BZ_DATA_ERROR: - if (zstream->log_errors) - bzlib_read_error(zstream, "corrupted data"); + bzlib_read_error(zstream, "corrupted data"); stream->istream.stream_errno = EINVAL; return -1; case BZ_DATA_ERROR_MAGIC: - if (zstream->log_errors) { - bzlib_read_error(zstream, - "wrong magic in header (not bz2 file?)"); - } + bzlib_read_error(zstream, + "wrong magic in header (not bz2 file?)"); stream->istream.stream_errno = EINVAL; return -1; case BZ_MEM_ERROR: diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-compression/istream-zlib.c --- a/src/lib-compression/istream-zlib.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-compression/istream-zlib.c Fri Sep 20 00:12:45 2013 +0300 @@ -55,10 +55,13 @@ static void zlib_read_error(struct zlib_istream *zstream, const char *error) { - i_error("zlib.read(%s): %s at %"PRIuUOFF_T, - i_stream_get_name(&zstream->istream.istream), error, - zstream->istream.abs_start_offset + - zstream->istream.istream.v_offset); + io_stream_set_error(&zstream->istream.iostream, + "zlib.read(%s): %s at %"PRIuUOFF_T, + i_stream_get_name(&zstream->istream.istream), error, + zstream->istream.abs_start_offset + + zstream->istream.istream.v_offset); + if (zstream->log_errors) + i_error("%s", zstream->istream.iostream.error); } static int i_stream_zlib_read_header(struct istream_private *stream) @@ -73,8 +76,7 @@ zstream->prev_size); if (size == zstream->prev_size) { if (ret == -1) { - if (zstream->log_errors) - zlib_read_error(zstream, "missing gz header"); + zlib_read_error(zstream, "missing gz header"); stream->istream.stream_errno = EINVAL; } return ret; @@ -87,10 +89,7 @@ if (data[0] != GZ_MAGIC1 || data[1] != GZ_MAGIC2) { /* missing gzip magic header */ - if (zstream->log_errors) { - zlib_read_error(zstream, "wrong magic in header " - "(not gz file?)"); - } + zlib_read_error(zstream, "wrong magic in header (not gz file?)"); stream->istream.stream_errno = EINVAL; return -1; } @@ -143,8 +142,7 @@ GZ_TRAILER_SIZE-1); if (size == zstream->prev_size) { if (ret == -1) { - if (zstream->log_errors) - zlib_read_error(zstream, "missing gz trailer"); + zlib_read_error(zstream, "missing gz trailer"); stream->istream.stream_errno = EINVAL; } return ret; @@ -155,10 +153,7 @@ return 0; if (data_get_uint32(data) != zstream->crc32) { - if (zstream->log_errors) { - zlib_read_error(zstream, - "gz trailer has wrong CRC value"); - } + zlib_read_error(zstream, "gz trailer has wrong CRC value"); stream->istream.stream_errno = EINVAL; return -1; } @@ -254,8 +249,7 @@ stream->parent->stream_errno; } else { i_assert(stream->parent->eof); - if (zstream->log_errors) - zlib_read_error(zstream, "unexpected EOF"); + zlib_read_error(zstream, "unexpected EOF"); stream->istream.stream_errno = EPIPE; } return -1; @@ -286,13 +280,11 @@ case Z_OK: break; case Z_NEED_DICT: - if (zstream->log_errors) - zlib_read_error(zstream, "can't read file without dict"); + zlib_read_error(zstream, "can't read file without dict"); stream->istream.stream_errno = EINVAL; return -1; case Z_DATA_ERROR: - if (zstream->log_errors) - zlib_read_error(zstream, "corrupted data"); + zlib_read_error(zstream, "corrupted data"); stream->istream.stream_errno = EINVAL; return -1; case Z_MEM_ERROR: diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-fs/istream-metawrap.c --- a/src/lib-fs/istream-metawrap.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-fs/istream-metawrap.c Fri Sep 20 00:12:45 2013 +0300 @@ -24,6 +24,8 @@ } p = strchr(line, ':'); if (p == NULL) { + io_stream_set_error(&mstream->istream.iostream, + "Metadata header line is missing ':'"); mstream->istream.istream.stream_errno = EINVAL; return -1; } diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-mail/istream-dot.c --- a/src/lib-mail/istream-dot.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-mail/istream-dot.c Fri Sep 20 00:12:45 2013 +0300 @@ -36,6 +36,8 @@ stream->parent->stream_errno; } else if (ret < 0 && stream->parent->eof) { /* we didn't see "." line */ + io_stream_set_error(&stream->iostream, + "dot-input stream ends without '.' line"); stream->istream.stream_errno = EPIPE; } return ret; diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-mail/istream-qp-decoder.c --- a/src/lib-mail/istream-qp-decoder.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-mail/istream-qp-decoder.c Fri Sep 20 00:12:45 2013 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "buffer.h" +#include "hex-binary.h" #include "istream-private.h" #include "quoted-printable.h" #include "istream-qp.h" @@ -64,6 +65,9 @@ ret = !eof ? quoted_printable_decode(data, size, &pos, &buf) : quoted_printable_decode_final(data, size, &pos, &buf); if (ret < 0) { + io_stream_set_error(&stream->iostream, + "Invalid quoted-printable data: 0x%s", + binary_to_hex(data+pos, I_MAX(size-pos, 8))); stream->istream.stream_errno = EINVAL; return -1; } @@ -77,7 +81,8 @@ { struct qp_decoder_istream *bstream = (struct qp_decoder_istream *)stream; - size_t pre_count, post_count; + const unsigned char *data; + size_t pre_count, post_count, size; int ret; size_t prev_size = 0; @@ -95,6 +100,10 @@ } /* partial qp input */ i_assert(ret < 0); + data = i_stream_get_data(stream->parent, &size); + io_stream_set_error(&stream->iostream, + "quoted-printable input ends with a partial block: 0x%s", + binary_to_hex(data, size)); stream->istream.stream_errno = EINVAL; return -1; } diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.c Fri Sep 20 00:12:45 2013 +0300 @@ -270,6 +270,7 @@ o_stream_unref(&ssl_io->plain_output); BIO_free(ssl_io->bio_ext); SSL_free(ssl_io->ssl); + i_free(ssl_io->plain_stream_errstr); i_free(ssl_io->last_error); i_free(ssl_io->host); i_free(ssl_io->log_prefix); @@ -331,6 +332,9 @@ if (sent < 0) { i_assert(ssl_io->plain_output->closed || ssl_io->plain_output->stream_errno != 0); + i_free(ssl_io->plain_stream_errstr); + ssl_io->plain_stream_errstr = + i_strdup(o_stream_get_error(ssl_io->plain_output)); ssl_io->plain_stream_errno = ssl_io->plain_output->stream_errno; ssl_io->closed = TRUE; @@ -376,6 +380,9 @@ ret = openssl_iostream_read_more(ssl_io, &data, &size); ssl_io->plain_input->real_stream->try_alloc_limit = 0; if (ret == -1 && size == 0 && !bytes_read) { + i_free(ssl_io->plain_stream_errstr); + ssl_io->plain_stream_errstr = + i_strdup(i_stream_get_error(ssl_io->plain_input)); ssl_io->plain_stream_errno = ssl_io->plain_input->stream_errno; ssl_io->closed = TRUE; @@ -397,12 +404,18 @@ if (bytes == 0 && !bytes_read && ssl_io->want_read) { /* shouldn't happen */ i_error("SSL BIO buffer size too small"); + i_free(ssl_io->plain_stream_errstr); + ssl_io->plain_stream_errstr = + i_strdup("SSL BIO buffer size too small"); ssl_io->plain_stream_errno = EINVAL; ssl_io->closed = TRUE; return FALSE; } if (i_stream_get_data_size(ssl_io->plain_input) > 0) { i_error("SSL: Too much data in buffered plain input buffer"); + i_free(ssl_io->plain_stream_errstr); + ssl_io->plain_stream_errstr = + i_strdup("SSL: Too much data in buffered plain input buffer"); ssl_io->plain_stream_errno = EINVAL; ssl_io->closed = TRUE; return FALSE; @@ -455,6 +468,8 @@ return 0; } if (ssl_io->closed) { + if (ssl_io->plain_stream_errstr != NULL) + openssl_iostream_set_error(ssl_io, ssl_io->plain_stream_errstr); errno = ssl_io->plain_stream_errno != 0 ? ssl_io->plain_stream_errno : EPIPE; return -1; @@ -464,6 +479,8 @@ ssl_io->want_read = TRUE; (void)openssl_iostream_bio_sync(ssl_io); if (ssl_io->closed) { + if (ssl_io->plain_stream_errstr != NULL) + openssl_iostream_set_error(ssl_io, ssl_io->plain_stream_errstr); errno = ssl_io->plain_stream_errno != 0 ? ssl_io->plain_stream_errno : EPIPE; return -1; @@ -489,7 +506,8 @@ case SSL_ERROR_ZERO_RETURN: /* clean connection closing */ errno = ECONNRESET; - break; + i_free_and_null(ssl_io->last_error); + return -1; case SSL_ERROR_SSL: errstr = t_strdup_printf("%s failed: %s", func_name, openssl_iostream_error()); @@ -503,8 +521,7 @@ break; } - if (errstr != NULL) - openssl_iostream_set_error(ssl_io, errstr); + openssl_iostream_set_error(ssl_io, errstr); return -1; } diff -r e35be66003e2 -r 76d5e3c8cec3 src/lib-ssl-iostream/iostream-openssl.h --- a/src/lib-ssl-iostream/iostream-openssl.h Fri Sep 20 00:00:49 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.h Fri Sep 20 00:12:45 2013 +0300 @@ -31,6 +31,7 @@ char *host; From dovecot at dovecot.org Fri Sep 20 00:14:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:14:23 +0300 Subject: dovecot-2.2: istream-file: Assert-crash if read() fails with EBADF. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/28df180ec3ab changeset: 16774:28df180ec3ab user: Timo Sirainen date: Fri Sep 20 00:14:11 2013 +0300 description: istream-file: Assert-crash if read() fails with EBADF. Something's already wrong at that point and it may not be safe to continue. Also crashing makes it easier to debug such situation. diffstat: src/lib/istream-file.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 76d5e3c8cec3 -r 28df180ec3ab src/lib/istream-file.c --- a/src/lib/istream-file.c Fri Sep 20 00:12:45 2013 +0300 +++ b/src/lib/istream-file.c Fri Sep 20 00:14:11 2013 +0300 @@ -94,6 +94,9 @@ ret = 0; } else { i_assert(errno != 0); + /* if we get EBADF for a valid fd, it means something's + really wrong and we'd better just crash. */ + i_assert(errno != EBADF); stream->istream.stream_errno = errno; return -1; } From dovecot at dovecot.org Fri Sep 20 00:20:36 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:20:36 +0300 Subject: dovecot-2.2: istream-chain/concat/seekable: When child stream fa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0eee8caeaa53 changeset: 16775:0eee8caeaa53 user: Timo Sirainen date: Fri Sep 20 00:20:19 2013 +0300 description: istream-chain/concat/seekable: When child stream fails, copy its error to ourself. diffstat: src/lib/istream-chain.c | 4 ++++ src/lib/istream-concat.c | 4 ++++ src/lib/istream-seekable.c | 4 ++++ 3 files changed, 12 insertions(+), 0 deletions(-) diffs (42 lines): diff -r 28df180ec3ab -r 0eee8caeaa53 src/lib/istream-chain.c --- a/src/lib/istream-chain.c Fri Sep 20 00:14:11 2013 +0300 +++ b/src/lib/istream-chain.c Fri Sep 20 00:20:19 2013 +0300 @@ -203,6 +203,10 @@ if (ret == -1) { if (link->stream->stream_errno != 0) { + io_stream_set_error(&stream->iostream, + "read(%s) failed: %s", + i_stream_get_name(link->stream), + i_stream_get_error(link->stream)); stream->istream.stream_errno = link->stream->stream_errno; return -1; diff -r 28df180ec3ab -r 0eee8caeaa53 src/lib/istream-concat.c --- a/src/lib/istream-concat.c Fri Sep 20 00:14:11 2013 +0300 +++ b/src/lib/istream-concat.c Fri Sep 20 00:20:19 2013 +0300 @@ -133,6 +133,10 @@ return ret; if (ret == -1 && cstream->cur_input->stream_errno != 0) { + io_stream_set_error(&cstream->istream.iostream, + "read(%s) failed: %s", + i_stream_get_name(cstream->cur_input), + i_stream_get_error(cstream->cur_input)); stream->istream.stream_errno = cstream->cur_input->stream_errno; return -1; diff -r 28df180ec3ab -r 0eee8caeaa53 src/lib/istream-seekable.c --- a/src/lib/istream-seekable.c Fri Sep 20 00:14:11 2013 +0300 +++ b/src/lib/istream-seekable.c Fri Sep 20 00:20:19 2013 +0300 @@ -139,6 +139,10 @@ while ((ret = i_stream_read(sstream->cur_input)) == -1) { if (sstream->cur_input->stream_errno != 0) { + io_stream_set_error(&sstream->istream.iostream, + "read(%s) failed: %s", + i_stream_get_name(sstream->cur_input), + i_stream_get_error(sstream->cur_input)); sstream->istream.istream.stream_errno = sstream->cur_input->stream_errno; return -1; From dovecot at dovecot.org Fri Sep 20 00:22:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:22:23 +0300 Subject: dovecot-2.2: imap: If FETCH fails, log the stream's error string... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c140de1b506b changeset: 16776:c140de1b506b user: Timo Sirainen date: Fri Sep 20 00:22:15 2013 +0300 description: imap: If FETCH fails, log the stream's error string instead of errno. diffstat: src/imap/cmd-append.c | 5 +++-- src/imap/cmd-urlfetch.c | 5 +++-- src/imap/imap-fetch-body.c | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diffs (44 lines): diff -r 0eee8caeaa53 -r c140de1b506b src/imap/cmd-append.c --- a/src/imap/cmd-append.c Fri Sep 20 00:20:19 2013 +0300 +++ b/src/imap/cmd-append.c Fri Sep 20 00:22:15 2013 +0300 @@ -212,8 +212,9 @@ if (mpresult.input->stream_errno != 0) { errno = mpresult.input->stream_errno; mail_storage_set_critical(ctx->box->storage, - "read(%s) failed: %m (for CATENATE URL %s)", - i_stream_get_name(mpresult.input), caturl); + "read(%s) failed: %s (for CATENATE URL %s)", + i_stream_get_name(mpresult.input), + i_stream_get_error(mpresult.input), caturl); client_send_storage_error(cmd, ctx->storage); ret = -1; } else if (!mpresult.input->eof) { diff -r 0eee8caeaa53 -r c140de1b506b src/imap/cmd-urlfetch.c --- a/src/imap/cmd-urlfetch.c Fri Sep 20 00:20:19 2013 +0300 +++ b/src/imap/cmd-urlfetch.c Fri Sep 20 00:22:15 2013 +0300 @@ -112,8 +112,9 @@ } if (ctx->input->stream_errno != 0) { errno = ctx->input->stream_errno; - i_error("read(%s) failed: %m (URLFETCH)", - i_stream_get_name(ctx->input)); + i_error("read(%s) failed: %s (URLFETCH)", + i_stream_get_name(ctx->input), + i_stream_get_error(ctx->input)); client_disconnect(client, "URLFETCH failed"); return -1; } diff -r 0eee8caeaa53 -r c140de1b506b src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Fri Sep 20 00:20:19 2013 +0300 +++ b/src/imap/imap-fetch-body.c Fri Sep 20 00:22:15 2013 +0300 @@ -33,8 +33,9 @@ errno = state->cur_input->stream_errno; mail_storage_set_critical(state->cur_mail->box->storage, - "read(%s) failed: %m (FETCH %s for mailbox %s UID %u)", + "read(%s) failed: %s (FETCH %s for mailbox %s UID %u)", i_stream_get_name(state->cur_input), + i_stream_get_error(state->cur_input), state->cur_human_name, mailbox_get_vname(state->cur_mail->box), state->cur_mail->uid); } From dovecot at dovecot.org Fri Sep 20 00:59:45 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 00:59:45 +0300 Subject: dovecot-2.2: ioloop-kqueue: Added assert Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c8b1e5833a28 changeset: 16777:c8b1e5833a28 user: Timo Sirainen date: Fri Sep 20 00:59:36 2013 +0300 description: ioloop-kqueue: Added assert diffstat: src/lib/ioloop-kqueue.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c140de1b506b -r c8b1e5833a28 src/lib/ioloop-kqueue.c --- a/src/lib/ioloop-kqueue.c Fri Sep 20 00:22:15 2013 +0300 +++ b/src/lib/ioloop-kqueue.c Fri Sep 20 00:59:36 2013 +0300 @@ -131,6 +131,7 @@ /* reference all IOs */ for (i = 0; i < ret; i++) { io = (void *)events[i].udata; + i_assert(io->refcount > 0); io->refcount++; } From dovecot at dovecot.org Fri Sep 20 02:28:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 02:28:59 +0300 Subject: dovecot-2.2: lib-auth: Don't leave stale pointers to stack lying... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ae2ae657a95 changeset: 16778:5ae2ae657a95 user: Timo Sirainen date: Fri Sep 20 02:28:45 2013 +0300 description: lib-auth: Don't leave stale pointers to stack lying around in memory. diffstat: src/lib-auth/auth-master.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (35 lines): diff -r c8b1e5833a28 -r 5ae2ae657a95 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Fri Sep 20 00:59:36 2013 +0300 +++ b/src/lib-auth/auth-master.c Fri Sep 20 02:28:45 2013 +0300 @@ -517,6 +517,7 @@ *username_r = ctx.fields[0]; *fields_r = ctx.fields + 1; } + conn->reply_context = NULL; return ctx.return_value; } @@ -583,6 +584,7 @@ *fields_r = ctx.fields != NULL ? ctx.fields : p_new(pool, const char *, 1); + conn->reply_context = NULL; return ctx.return_value; } @@ -625,6 +627,7 @@ (void)auth_master_run_cmd(conn, str_c(str)); conn->prefix = DEFAULT_USERDB_LOOKUP_PREFIX; + conn->reply_context = NULL; return *count_r == UINT_MAX ? -1 : 0; } @@ -687,6 +690,7 @@ ctx->failed = TRUE; io_loop_set_current(conn->prev_ioloop); conn->prefix = DEFAULT_USERDB_LOOKUP_PREFIX; + conn->reply_context = NULL; return ctx; } From dovecot at dovecot.org Fri Sep 20 02:39:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 02:39:07 +0300 Subject: dovecot-2.2: mbox: Handle correctly if MAIL_FETCH_HEADER_MD5 loo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f9ab3d21689b changeset: 16779:f9ab3d21689b user: Timo Sirainen date: Fri Sep 20 02:38:53 2013 +0300 description: mbox: Handle correctly if MAIL_FETCH_HEADER_MD5 lookup fails because mail is already expunged. diffstat: src/lib-storage/index/mbox/mbox-mail.c | 26 +++++++++++++++----------- 1 files changed, 15 insertions(+), 11 deletions(-) diffs (74 lines): diff -r 5ae2ae657a95 -r f9ab3d21689b src/lib-storage/index/mbox/mbox-mail.c --- a/src/lib-storage/index/mbox/mbox-mail.c Fri Sep 20 02:28:45 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-mail.c Fri Sep 20 02:38:53 2013 +0300 @@ -152,26 +152,29 @@ static bool mbox_mail_get_md5_header(struct index_mail *mail, const char **value_r) { + struct mail *_mail = &mail->mail.mail; static uint8_t empty_md5[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - struct mbox_mailbox *mbox = (struct mbox_mailbox *)mail->mail.mail.box; + struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box; const void *ext_data; if (mail->data.guid != NULL) { *value_r = mail->data.guid; - return TRUE; + return 1; } - mail_index_lookup_ext(mail->mail.mail.transaction->view, - mail->mail.mail.seq, mbox->md5hdr_ext_idx, - &ext_data, NULL); + mail_index_lookup_ext(_mail->transaction->view, _mail->seq, + mbox->md5hdr_ext_idx, &ext_data, NULL); if (ext_data != NULL && memcmp(ext_data, empty_md5, 16) != 0) { mail->data.guid = p_strdup(mail->mail.data_pool, binary_to_hex(ext_data, 16)); *value_r = mail->data.guid; - return TRUE; + return 1; + } else if (mail_index_is_expunged(_mail->transaction->view, _mail->seq)) { + mail_set_expunged(_mail); + return -1; } else { - return FALSE; + return 0; } } @@ -183,6 +186,7 @@ struct mbox_mailbox *mbox = (struct mbox_mailbox *)_mail->box; uoff_t offset; bool move_offset; + int ret; switch (field) { case MAIL_FETCH_FROM_ENVELOPE: @@ -193,8 +197,8 @@ return 0; case MAIL_FETCH_GUID: case MAIL_FETCH_HEADER_MD5: - if (mbox_mail_get_md5_header(mail, value_r)) - return 0; + if ((ret = mbox_mail_get_md5_header(mail, value_r)) != 0) + return ret < 0 ? -1 : 0; /* i guess in theory the empty_md5 is valid and can happen, but it's almost guaranteed that it means the MD5 sum is @@ -220,12 +224,12 @@ } } - if (!mbox_mail_get_md5_header(mail, value_r)) { + if ((ret = mbox_mail_get_md5_header(mail, value_r)) == 0) { i_error("mbox %s resyncing didn't save header MD5 values", _mail->box->name); return -1; } - return 0; + return ret < 0 ? -1 : 0; default: break; } From dovecot at dovecot.org Fri Sep 20 03:42:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 03:42:04 +0300 Subject: dovecot-2.2: master: Make sure new processes aren't created afte... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d7627178a7f2 changeset: 16780:d7627178a7f2 user: Timo Sirainen date: Fri Sep 20 03:41:51 2013 +0300 description: master: Make sure new processes aren't created after stop signal is received. diffstat: src/master/main.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r f9ab3d21689b -r d7627178a7f2 src/master/main.c --- a/src/master/main.c Fri Sep 20 02:38:53 2013 +0300 +++ b/src/master/main.c Fri Sep 20 03:41:51 2013 +0300 @@ -438,6 +438,9 @@ si->si_signo, dec2str(si->si_pid), dec2str(si->si_uid), lib_signal_code_to_str(si->si_signo, si->si_code)); + /* make sure new processes won't be created by the currently + running ioloop. */ + services->destroying = TRUE; master_service_stop(master_service); } From dovecot at dovecot.org Fri Sep 20 03:54:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 03:54:49 +0300 Subject: dovecot-2.2: dbox: Fixed "UIDVALIDITY=0" error race condition. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/98702a45784c changeset: 16781:98702a45784c user: Timo Sirainen date: Fri Sep 20 03:54:31 2013 +0300 description: dbox: Fixed "UIDVALIDITY=0" error race condition. If session 1 had mkdir()ed but not yet created the initial index, while session 2 attempted to open the mailbox, it would create an empty index and log the error. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage.c | 6 +++--- src/lib-storage/index/dbox-multi/mdbox-storage.h | 3 +++ src/lib-storage/index/dbox-multi/mdbox-sync.c | 10 +++++++++- src/lib-storage/index/dbox-single/sdbox-storage.c | 7 +++---- src/lib-storage/index/dbox-single/sdbox-storage.h | 3 +++ src/lib-storage/index/dbox-single/sdbox-sync.c | 8 ++++++++ 6 files changed, 29 insertions(+), 8 deletions(-) diffs (98 lines): diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Fri Sep 20 03:54:31 2013 +0300 @@ -319,9 +319,9 @@ return 0; } -static int mdbox_mailbox_create_indexes(struct mailbox *box, - const struct mailbox_update *update, - struct mail_index_transaction *trans) +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) { struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; int ret; diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-multi/mdbox-storage.h --- a/src/lib-storage/index/dbox-multi/mdbox-storage.h Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.h Fri Sep 20 03:54:31 2013 +0300 @@ -75,6 +75,9 @@ void mdbox_update_header(struct mdbox_mailbox *mbox, struct mail_index_transaction *trans, const struct mailbox_update *update) ATTR_NULL(3); +int mdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); struct mail_save_context * mdbox_save_alloc(struct mailbox_transaction_context *_t); diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-multi/mdbox-sync.c --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Fri Sep 20 03:54:31 2013 +0300 @@ -137,8 +137,16 @@ hdr = mail_index_get_header(ctx->sync_view); if (hdr->uid_validity == 0) { /* newly created index file */ + if (hdr->next_uid == 1) { + /* could be just a race condition where we opened the + mailbox between mkdir and index creation. fix this + silently. */ + if (mdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0) + return -1; + return 1; + } mail_storage_set_critical(box->storage, - "Mailbox %s: Corrupted index, uidvalidity=0", + "Mailbox %s: Broken index: missing UIDVALIDITY", box->vname); return 0; } diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Fri Sep 20 03:54:31 2013 +0300 @@ -211,10 +211,9 @@ sizeof(mbox->mailbox_guid)); } -static int ATTR_NULL(2, 3) -sdbox_mailbox_create_indexes(struct mailbox *box, - const struct mailbox_update *update, - struct mail_index_transaction *trans) +int sdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) { struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; struct mail_index_transaction *new_trans = NULL; diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-single/sdbox-storage.h --- a/src/lib-storage/index/dbox-single/sdbox-storage.h Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.h Fri Sep 20 03:54:31 2013 +0300 @@ -41,6 +41,9 @@ int sdbox_read_header(struct sdbox_mailbox *mbox, struct sdbox_index_header *hdr, bool log_error, bool *need_resize_r); +int sdbox_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans); void sdbox_set_mailbox_corrupted(struct mailbox *box); struct mail_save_context * diff -r d7627178a7f2 -r 98702a45784c src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Fri Sep 20 03:41:51 2013 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Fri Sep 20 03:54:31 2013 +0300 @@ -107,6 +107,14 @@ hdr = mail_index_get_header(ctx->sync_view); if (hdr->uid_validity == 0) { /* newly created index file */ + if (hdr->next_uid == 1) { + /* could be just a race condition where we opened the + mailbox between mkdir and index creation. fix this + silently. */ + if (sdbox_mailbox_create_indexes(box, NULL, ctx->trans) < 0) + return -1; + return 1; + } mail_storage_set_critical(box->storage, "sdbox %s: Broken index: missing UIDVALIDITY", mailbox_get_path(box)); From dovecot at dovecot.org Fri Sep 20 04:01:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 04:01:25 +0300 Subject: dovecot-2.2: lib-ssl-iostream: Give better error message if CA s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac832f051b49 changeset: 16782:ac832f051b49 user: Timo Sirainen date: Fri Sep 20 04:01:10 2013 +0300 description: lib-ssl-iostream: Give better error message if CA settings are missing. diffstat: src/lib-ssl-iostream/iostream-openssl-context.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 98702a45784c -r ac832f051b49 src/lib-ssl-iostream/iostream-openssl-context.c --- a/src/lib-ssl-iostream/iostream-openssl-context.c Fri Sep 20 03:54:31 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl-context.c Fri Sep 20 04:01:10 2013 +0300 @@ -349,7 +349,9 @@ } if (!have_ca) { - *error_r = "Can't verify remote certs without CA"; + *error_r = !ctx->client_ctx ? + "Can't verify remote client certs without CA (ssl_ca setting)" : + "Can't verify remote server certs without trusted CAs (ssl_client_ca_* settings)"; return -1; } return 0; From dovecot at dovecot.org Fri Sep 20 04:20:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 04:20:48 +0300 Subject: dovecot-2.2: ssl-params: Long-running ssl-params process shouldn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/17aaab5511f2 changeset: 16783:17aaab5511f2 user: Timo Sirainen date: Fri Sep 20 04:20:22 2013 +0300 description: ssl-params: Long-running ssl-params process shouldn't cause Dovecot restart to fail. diffstat: src/ssl-params/ssl-params.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r ac832f051b49 -r 17aaab5511f2 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Fri Sep 20 04:01:10 2013 +0300 +++ b/src/ssl-params/ssl-params.c Fri Sep 20 04:20:22 2013 +0300 @@ -5,6 +5,7 @@ #include "buffer.h" #include "file-lock.h" #include "read-full.h" +#include "master-service.h" #include "master-service-settings.h" #include "ssl-params-settings.h" #include "ssl-params.h" @@ -118,7 +119,9 @@ case -1: i_fatal("fork() failed: %m"); case 0: - /* child */ + /* child - close listener fds so a long-running ssl-params + doesn't cause Dovecot restart to fail */ + master_service_stop_new_connections(master_service); ssl_params_if_unchanged(param->path, param->last_mtime); exit(0); default: From dovecot at dovecot.org Fri Sep 20 04:21:40 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 04:21:40 +0300 Subject: dovecot-2.1: ssl-params: Long-running ssl-params process shouldn... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/dc631b8b1290 changeset: 14998:dc631b8b1290 user: Timo Sirainen date: Fri Sep 20 04:20:22 2013 +0300 description: ssl-params: Long-running ssl-params process shouldn't cause Dovecot restart to fail. diffstat: src/ssl-params/ssl-params.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 9efbc0731929 -r dc631b8b1290 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Tue Aug 27 04:11:24 2013 +0300 +++ b/src/ssl-params/ssl-params.c Fri Sep 20 04:20:22 2013 +0300 @@ -5,6 +5,7 @@ #include "buffer.h" #include "file-lock.h" #include "read-full.h" +#include "master-service.h" #include "master-service-settings.h" #include "ssl-params-settings.h" #include "ssl-params.h" @@ -118,7 +119,9 @@ case -1: i_fatal("fork() failed: %m"); case 0: - /* child */ + /* child - close listener fds so a long-running ssl-params + doesn't cause Dovecot restart to fail */ + master_service_stop_new_connections(master_service); ssl_params_if_unchanged(param->path, param->last_mtime); exit(0); default: From dovecot at dovecot.org Fri Sep 20 04:27:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 04:27:44 +0300 Subject: dovecot-2.2: dsync: Don't log "Mailbox changes caused a desync" ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3da7f0dad08f changeset: 16784:3da7f0dad08f user: Timo Sirainen date: Fri Sep 20 04:27:29 2013 +0300 description: dsync: Don't log "Mailbox changes caused a desync" warning when running via doveadm-server. diffstat: src/doveadm/dsync/doveadm-dsync.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 17aaab5511f2 -r 3da7f0dad08f src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Fri Sep 20 04:20:22 2013 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Fri Sep 20 04:27:29 2013 +0300 @@ -576,8 +576,12 @@ } if (dsync_brain_has_unexpected_changes(brain) || changes_during_sync) { - i_warning("Mailbox changes caused a desync. " - "You may want to run dsync again."); + /* don't log a warning when running via doveadm server + (e.g. called by replicator) */ + if (ctx->ctx.conn == NULL) { + i_warning("Mailbox changes caused a desync. " + "You may want to run dsync again."); + } ctx->ctx.exit_code = 2; } if (dsync_brain_deinit(&brain) < 0) { From dovecot at dovecot.org Fri Sep 20 09:46:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 09:46:48 +0300 Subject: dovecot-2.2: director: Don't assert-crash if PASS lookup fails. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b135d723acf3 changeset: 16785:b135d723acf3 user: Timo Sirainen date: Fri Sep 20 09:46:31 2013 +0300 description: director: Don't assert-crash if PASS lookup fails. diffstat: src/director/login-connection.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (31 lines): diff -r 3da7f0dad08f -r b135d723acf3 src/director/login-connection.c --- a/src/director/login-connection.c Fri Sep 20 04:27:29 2013 +0300 +++ b/src/director/login-connection.c Fri Sep 20 09:46:31 2013 +0300 @@ -77,7 +77,7 @@ { struct login_host_request *request = context; struct director *dir = request->conn->dir; - const char *line; + const char *line, *line_params; unsigned int secs; if (ip != NULL) { @@ -85,11 +85,16 @@ line = t_strdup_printf("%s\thost=%s\tproxy_refresh=%u", request->line, net_ip2addr(ip), secs); } else { - i_assert(strncmp(request->line, "OK\t", 3) == 0); + if (strncmp(request->line, "OK\t", 3) == 0) + line_params = request->line + 3; + else if (strncmp(request->line, "PASS\t", 5) == 0) + line_params = request->line + 5; + else + i_panic("BUG: Unexpected line: %s", request->line); i_error("director: User %s host lookup failed: %s", request->username, errormsg); - line = t_strconcat("FAIL\t", t_strcut(request->line + 3, '\t'), + line = t_strconcat("FAIL\t", t_strcut(line_params, '\t'), "\ttemp", NULL); } login_connection_send_line(request->conn, line); From dovecot at dovecot.org Fri Sep 20 10:01:02 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 10:01:02 +0300 Subject: dovecot-2.2: director: Added more debug logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/17389ac3cb66 changeset: 16786:17389ac3cb66 user: Timo Sirainen date: Fri Sep 20 10:00:48 2013 +0300 description: director: Added more debug logging. diffstat: src/director/director-connection.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (27 lines): diff -r b135d723acf3 -r 17389ac3cb66 src/director/director-connection.c --- a/src/director/director-connection.c Fri Sep 20 09:46:31 2013 +0300 +++ b/src/director/director-connection.c Fri Sep 20 10:00:48 2013 +0300 @@ -1122,15 +1122,23 @@ } else { if (seq < host->last_sync_seq) { /* stale SYNC event */ + dir_debug("Ignore stale SYNC event for %s (seq %u < %u)", + host->name, seq, host->last_sync_seq); return FALSE; } else if (host->last_sync_seq != seq || timestamp > host->last_sync_timestamp) { host->last_sync_seq = seq; host->last_sync_timestamp = timestamp; host->last_sync_seq_counter = 1; + dir_debug("Update SYNC for %s (seq=%u, timestamp=%u)", + host->name, seq, timestamp); } else if (++host->last_sync_seq_counter > DIRECTOR_MAX_SYNC_SEQ_DUPLICATES) { /* we've received this too many times already */ + dir_debug("Ignore duplicate #%u SYNC event for %s " + "(seq=%u, timestamp %u <= %u)", + host->name, host->last_sync_seq_counter, seq, + timestamp, host->last_sync_timestamp); return FALSE; } From dovecot at dovecot.org Fri Sep 20 10:12:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 10:12:38 +0300 Subject: dovecot-2.2: director: Fix & improvement to debug logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22a3ef9a006d changeset: 16787:22a3ef9a006d user: Timo Sirainen date: Fri Sep 20 10:11:45 2013 +0300 description: director: Fix & improvement to debug logging. diffstat: src/director/director-connection.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (25 lines): diff -r 17389ac3cb66 -r 22a3ef9a006d src/director/director-connection.c --- a/src/director/director-connection.c Fri Sep 20 10:00:48 2013 +0300 +++ b/src/director/director-connection.c Fri Sep 20 10:11:45 2013 +0300 @@ -1122,8 +1122,10 @@ } else { if (seq < host->last_sync_seq) { /* stale SYNC event */ - dir_debug("Ignore stale SYNC event for %s (seq %u < %u)", - host->name, seq, host->last_sync_seq); + dir_debug("Ignore stale SYNC event for %s " + "(seq %u < %u, timestamp=%u)", + host->name, seq, host->last_sync_seq, + timestamp); return FALSE; } else if (host->last_sync_seq != seq || timestamp > host->last_sync_timestamp) { @@ -1137,7 +1139,7 @@ /* we've received this too many times already */ dir_debug("Ignore duplicate #%u SYNC event for %s " "(seq=%u, timestamp %u <= %u)", - host->name, host->last_sync_seq_counter, seq, + host->last_sync_seq_counter, host->name, seq, timestamp, host->last_sync_timestamp); return FALSE; } From dovecot at dovecot.org Fri Sep 20 10:12:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 10:12:38 +0300 Subject: dovecot-2.2: director: Directors weren't always marked as restar... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b78c705bbb8d changeset: 16788:b78c705bbb8d user: Timo Sirainen date: Fri Sep 20 10:12:24 2013 +0300 description: director: Directors weren't always marked as restarted when they were. diffstat: src/director/director-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 22a3ef9a006d -r b78c705bbb8d src/director/director-connection.c --- a/src/director/director-connection.c Fri Sep 20 10:11:45 2013 +0300 +++ b/src/director/director-connection.c Fri Sep 20 10:12:24 2013 +0300 @@ -644,8 +644,8 @@ /* already have this. just reset its last_network_failure timestamp, since it might be up now. */ host->last_network_failure = 0; - if (host->last_seq != 0) { - /* it also may have been restarted, reset last_seq */ + if (host->last_seq != 0 || host->last_sync_seq != 0) { + /* it also may have been restarted, reset its state */ director_host_restarted(host); forward = TRUE; } From dovecot at dovecot.org Fri Sep 20 10:45:41 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 10:45:41 +0300 Subject: dovecot-2.2: director: Make sure director restart notifications ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d43c926eb97 changeset: 16789:5d43c926eb97 user: Timo Sirainen date: Fri Sep 20 10:35:34 2013 +0300 description: director: Make sure director restart notifications go to everyone in the ring. diffstat: src/director/director-connection.c | 19 ++++++++----------- 1 files changed, 8 insertions(+), 11 deletions(-) diffs (40 lines): diff -r b78c705bbb8d -r 5d43c926eb97 src/director/director-connection.c --- a/src/director/director-connection.c Fri Sep 20 10:12:24 2013 +0300 +++ b/src/director/director-connection.c Fri Sep 20 10:35:34 2013 +0300 @@ -625,7 +625,6 @@ struct director_host *host; struct ip_addr ip; unsigned int port; - bool forward = FALSE; if (!director_args_parse_ip_port(conn, args, &ip, &port)) return FALSE; @@ -644,20 +643,18 @@ /* already have this. just reset its last_network_failure timestamp, since it might be up now. */ host->last_network_failure = 0; - if (host->last_seq != 0 || host->last_sync_seq != 0) { - /* it also may have been restarted, reset its state */ - director_host_restarted(host); - forward = TRUE; - } + /* it also may have been restarted, reset its state */ + director_host_restarted(host); } else { /* save the director and forward it */ host = director_host_add(conn->dir, &ip, port); - forward = TRUE; } - if (forward) { - director_notify_ring_added(host, - director_connection_get_host(conn)); - } + /* just forward this to the entire ring until it reaches back to + itself. some hosts may see this twice, but that's the only way to + guarantee that it gets seen by everyone. reseting the host multiple + times may cause us to handle its commands multiple times, but the + commands can handle that. */ + director_notify_ring_added(host, director_connection_get_host(conn)); return TRUE; } From dovecot at dovecot.org Fri Sep 20 10:45:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 20 Sep 2013 10:45:42 +0300 Subject: dovecot-2.2: director: Detect lost director restarts and reset l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c30453a58b4c changeset: 16790:c30453a58b4c user: Timo Sirainen date: Fri Sep 20 10:45:22 2013 +0300 description: director: Detect lost director restarts and reset last_sync_seq. diffstat: src/director/director-connection.c | 24 ++++++++++++++++++++---- 1 files changed, 20 insertions(+), 4 deletions(-) diffs (51 lines): diff -r 5d43c926eb97 -r c30453a58b4c src/director/director-connection.c --- a/src/director/director-connection.c Fri Sep 20 10:35:34 2013 +0300 +++ b/src/director/director-connection.c Fri Sep 20 10:45:22 2013 +0300 @@ -74,6 +74,10 @@ #define DIRECTOR_HANDSHAKE_WARN_SECS 29 #define DIRECTOR_HANDSHAKE_BYTES_LOG_MIN_SECS (60*30) #define DIRECTOR_MAX_SYNC_SEQ_DUPLICATES 4 +/* If we receive SYNCs with a timestamp this many seconds higher than the last + valid received SYNC timestamp, assume that we lost the director's restart + notification and reset the last_sync_seq */ +#define DIRECTOR_SYNC_STALE_TIMESTAMP_RESET_SECS (60*2) #if DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS <= DIRECTOR_CONNECTION_PING_TIMEOUT_MSECS # error DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS is too low @@ -1117,20 +1121,32 @@ director_set_ring_synced(dir); } } else { - if (seq < host->last_sync_seq) { + if (seq < host->last_sync_seq && + timestamp < host->last_sync_timestamp + + DIRECTOR_SYNC_STALE_TIMESTAMP_RESET_SECS) { /* stale SYNC event */ dir_debug("Ignore stale SYNC event for %s " "(seq %u < %u, timestamp=%u)", host->name, seq, host->last_sync_seq, timestamp); return FALSE; - } else if (host->last_sync_seq != seq || + } else if (seq < host->last_sync_seq) { + i_warning("Last SYNC seq for %s appears to be stale, reseting " + "(seq=%u, timestamp=%u -> seq=%u, timestamp=%u)", + host->name, host->last_sync_seq, + host->last_sync_timestamp, seq, timestamp); + host->last_sync_seq = seq; + host->last_sync_timestamp = timestamp; + host->last_sync_seq_counter = 1; + } else if (seq > host->last_sync_seq || timestamp > host->last_sync_timestamp) { host->last_sync_seq = seq; host->last_sync_timestamp = timestamp; host->last_sync_seq_counter = 1; - dir_debug("Update SYNC for %s (seq=%u, timestamp=%u)", - host->name, seq, timestamp); + dir_debug("Update SYNC for %s " + "(seq=%u, timestamp=%u -> seq=%u, timestamp=%u)", + host->name, host->last_sync_seq, + host->last_sync_timestamp, seq, timestamp); } else if (++host->last_sync_seq_counter > DIRECTOR_MAX_SYNC_SEQ_DUPLICATES) { /* we've received this too many times already */ From dovecot at dovecot.org Sat Sep 21 02:55:45 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 02:55:45 +0300 Subject: dovecot-2.2: imapc: Don't log unnecessary errors at deinit about... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc1ee7b50ca9 changeset: 16791:cc1ee7b50ca9 user: Timo Sirainen date: Sat Sep 21 02:55:27 2013 +0300 description: imapc: Don't log unnecessary errors at deinit about aborted hiearchy separator lookup. The lookup is done asynchronously at startup. It doesn't matter if we never get around to finishing it. diffstat: src/lib-storage/index/imapc/imapc-list.c | 2 +- src/lib-storage/mail-user.c | 2 ++ src/lib-storage/mail-user.h | 2 ++ 3 files changed, 5 insertions(+), 1 deletions(-) diffs (36 lines): diff -r c30453a58b4c -r cc1ee7b50ca9 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Fri Sep 20 10:45:22 2013 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Sat Sep 21 02:55:27 2013 +0300 @@ -268,7 +268,7 @@ imapc_list_sep_verify(list); else if (reply->state == IMAPC_COMMAND_STATE_NO) imapc_list_copy_error_from_reply(list, MAIL_ERROR_PARAMS, reply); - else { + else if (!list->list.ns->user->deinitializing) { mailbox_list_set_critical(&list->list, "imapc: Command failed: %s", reply->text_full); } diff -r c30453a58b4c -r cc1ee7b50ca9 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Fri Sep 20 10:45:22 2013 +0300 +++ b/src/lib-storage/mail-user.c Sat Sep 21 02:55:27 2013 +0300 @@ -156,6 +156,8 @@ return; } + user->deinitializing = TRUE; + /* call deinit() with refcount=1, otherwise we may assert-crash in mail_user_ref() that is called by some deinit() handler. */ user->v.deinit(user); diff -r c30453a58b4c -r cc1ee7b50ca9 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Fri Sep 20 10:45:22 2013 +0300 +++ b/src/lib-storage/mail-user.h Sat Sep 21 02:55:27 2013 +0300 @@ -68,6 +68,8 @@ unsigned int dsyncing:1; /* Failed to create attribute dict, don't try again */ unsigned int attr_dict_failed:1; + /* We're deinitializing the user */ + unsigned int deinitializing:1; }; struct mail_user_module_register { From dovecot at dovecot.org Sat Sep 21 03:29:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 03:29:23 +0300 Subject: dovecot-2.2: dsync: Added some more debug logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a969a2e9399 changeset: 16792:4a969a2e9399 user: Timo Sirainen date: Sat Sep 21 03:29:09 2013 +0300 description: dsync: Added some more debug logging. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 47 ++++++++++++++++++++++++++++++++ 1 files changed, 47 insertions(+), 0 deletions(-) diffs (116 lines): diff -r cc1ee7b50ca9 -r 4a969a2e9399 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Sat Sep 21 02:55:27 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Sat Sep 21 03:29:09 2013 +0300 @@ -3,9 +3,11 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "str.h" #include "hex-binary.h" #include "istream.h" #include "seq-range-array.h" +#include "imap-util.h" #include "mail-storage-private.h" #include "mail-search-build.h" #include "dsync-transaction-log-scan.h" @@ -112,6 +114,21 @@ bool final); static void +imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...) +{ + va_list args; + + if (importer->debug) T_BEGIN { + va_start(args, fmt); + i_debug("brain %c: Import %s: %s", + importer->master_brain ? 'M' : 'S', + mailbox_get_vname(importer->box), + t_strdup_vprintf(fmt, args)); + va_end(args); + } T_END; +} + +static void dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer) { struct mail_search_args *search_args; @@ -1243,6 +1260,8 @@ unsigned int n, i, count; uint32_t uid; + imp_debug(importer, "Last common UID=%u", importer->last_common_uid); + importer->last_common_uid_found = TRUE; dsync_mailbox_rewind_search(importer); @@ -1395,6 +1414,10 @@ if (importer->failed) return -1; + imp_debug(importer, "Import change GUID=%s UID=%u hdr_hash=%s", + change->guid != NULL ? change->guid : "", change->uid, + change->hdr_hash != NULL ? change->hdr_hash : ""); + if (!importer->last_common_uid_found) { dsync_mailbox_find_common_uid(importer, change); if (importer->failed) @@ -1997,6 +2020,9 @@ if (importer->failed) return; + imp_debug(importer, "Import mail body for GUID=%s UID=%u", + mail->guid, mail->uid); + all_newmails = *mail->guid != '\0' ? hash_table_lookup(importer->import_guids, mail->guid) : hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid)); @@ -2006,6 +2032,9 @@ "GUID=%s UID=%u", mailbox_get_vname(importer->box), mail->guid, mail->uid); + } else { + imp_debug(importer, "Skip unwanted mail body for " + "GUID=%s UID=%u", mail->guid, mail->uid); } return; } @@ -2039,6 +2068,12 @@ if (array_count(unwanted_uids) == 0) return 1; + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, unwanted_uids); + imp_debug(importer, "Reassign UIDs: %s", str_c(str)); + } T_END; + search_args = mail_search_build_init(); arg = mail_search_build_add(search_args, SEARCH_UIDSET); p_array_init(&arg->value.seqset, search_args->pool, @@ -2164,6 +2199,11 @@ ret = -1; } else { /* remember the UIDs that were successfully saved */ + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, &changes.saved_uids); + imp_debug(importer, "Saved UIDs: %s", str_c(str)); + } T_END; seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0; while (seq_range_array_iter_nth(&iter, n++, &uid)) array_append(&importer->saved_uids, &uid, 1); @@ -2202,6 +2242,13 @@ update.min_highest_modseq = importer->remote_highest_modseq; update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq; + imp_debug(importer, "Finish update: min_next_uid=%u " + "min_first_recent_uid=%u min_highest_modseq=%llu " + "min_highest_pvt_modseq=%llu", + update.min_next_uid, update.min_first_recent_uid, + update.min_highest_modseq, + update.min_highest_pvt_modseq); + if (mailbox_update(importer->box, &update) < 0) { i_error("Mailbox %s: Update failed: %s", mailbox_get_vname(importer->box), From dovecot at dovecot.org Sat Sep 21 03:41:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 03:41:07 +0300 Subject: dovecot-2.2: dsync: Added more consistency when debug logging ab... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ce07adab319 changeset: 16793:2ce07adab319 user: Timo Sirainen date: Sat Sep 21 03:40:46 2013 +0300 description: dsync: Added more consistency when debug logging about changes during sync. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 11 ++++++----- src/doveadm/dsync/dsync-brain-mailbox.c | 4 ++-- src/doveadm/dsync/dsync-mailbox-import.c | 9 +++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diffs (68 lines): diff -r 4a969a2e9399 -r 2ce07adab319 src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Sat Sep 21 03:29:09 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Sat Sep 21 03:40:46 2013 +0300 @@ -124,8 +124,8 @@ } if (ret == 0) { if (brain->debug) { - i_debug("brain %c: Mailbox GUID %s sync: " - "Deletion conflict: %s", + i_debug("brain %c: Change during sync: " + "Mailbox GUID %s deletion conflict: %s", brain->master_brain ? 'M' : 'S', mailbox_get_vname(box), errstr); } @@ -143,8 +143,8 @@ if (error == MAIL_ERROR_NOTFOUND || error == MAIL_ERROR_EXISTS) { if (brain->debug) { - i_debug("brain %c: Mailbox %s sync: " - "mailbox_list_delete_dir conflict: %s", + i_debug("brain %c: Change during sync: " + "Mailbox %s mailbox_list_delete_dir conflict: %s", brain->master_brain ? 'M' : 'S', mailbox_get_vname(box), errstr); } @@ -198,7 +198,8 @@ /* mailbox was already created or was already deleted. let the next sync figure out what to do */ if (brain->debug) { - i_debug("brain %c: Mailbox %s sync: %s conflict: %s", + i_debug("brain %c: Change during sync: " + "Mailbox %s %s conflict: %s", brain->master_brain ? 'M' : 'S', mailbox_get_vname(box), func_name, errstr); diff -r 4a969a2e9399 -r 2ce07adab319 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sat Sep 21 03:29:09 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sat Sep 21 03:40:46 2013 +0300 @@ -625,8 +625,8 @@ /* mailbox was probably deleted/renamed during sync */ //FIXME: verify this from log, and if not log an error. if (brain->debug) { - i_debug("brain %c: Mailbox GUID %s sync: " - "Mailbox was lost during sync", + i_debug("brain %c: Change during sync: " + "Mailbox GUID %s was lost", brain->master_brain ? 'M' : 'S', guid_128_to_string(dsync_box->mailbox_guid)); } diff -r 4a969a2e9399 -r 2ce07adab319 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Sat Sep 21 03:29:09 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Sat Sep 21 03:40:46 2013 +0300 @@ -2110,10 +2110,11 @@ mailbox_get_last_error(box, NULL)); ret = -1; } - if (ret == 0 && importer->debug) { - i_debug("Mailbox %s: Renumbered %u of %u unwanted UIDs", - mailbox_get_vname(box), - renumber_count, array_count(unwanted_uids)); + if (ret == 0) { + imp_debug(importer, "Mailbox %s: Change during sync: " + "Renumbered %u of %u unwanted UIDs", + mailbox_get_vname(box), + renumber_count, array_count(unwanted_uids)); } return ret; } From dovecot at dovecot.org Sat Sep 21 03:49:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 03:49:42 +0300 Subject: dovecot-2.2: doveadm sync: When -1 parameter is used, ignore mis... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ec0dc639a54 changeset: 16794:0ec0dc639a54 user: Timo Sirainen date: Sat Sep 21 03:49:22 2013 +0300 description: doveadm sync: When -1 parameter is used, ignore missing mailboxes in source. Previously the syncs finished with "Mailbox changes caused a desync." and exit code 2. This was especially common when the destination server had new autocreated mailboxes and user logged in after the first dsync. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 2ce07adab319 -r 0ec0dc639a54 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sat Sep 21 03:40:46 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sat Sep 21 03:49:22 2013 +0300 @@ -623,6 +623,16 @@ } if (box == NULL) { /* mailbox was probably deleted/renamed during sync */ + if (brain->backup_send && brain->no_backup_overwrite) { + if (brain->debug) { + i_debug("brain %c: Ignore nonexistent " + "mailbox GUID %s with -1 sync", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_brain_slave_send_mailbox_lost(brain, dsync_box); + return TRUE; + } //FIXME: verify this from log, and if not log an error. if (brain->debug) { i_debug("brain %c: Change during sync: " From dovecot at dovecot.org Sat Sep 21 05:03:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 05:03:25 +0300 Subject: dovecot-2.2: lib-auth: Recent cleanup commit was actually cleani... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a4ae8d511a2 changeset: 16795:7a4ae8d511a2 user: Timo Sirainen date: Sat Sep 21 04:02:51 2013 +0300 description: lib-auth: Recent cleanup commit was actually cleaning up too much. diffstat: src/lib-auth/auth-master.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 0ec0dc639a54 -r 7a4ae8d511a2 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Sat Sep 21 03:49:22 2013 +0300 +++ b/src/lib-auth/auth-master.c Sat Sep 21 04:02:51 2013 +0300 @@ -690,7 +690,6 @@ ctx->failed = TRUE; io_loop_set_current(conn->prev_ioloop); conn->prefix = DEFAULT_USERDB_LOOKUP_PREFIX; - conn->reply_context = NULL; return ctx; } From dovecot at dovecot.org Sat Sep 21 05:03:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 05:03:25 +0300 Subject: dovecot-2.2: lib-auth: Fixed infinite looping if user listing fa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/09f5e8d6b8d4 changeset: 16796:09f5e8d6b8d4 user: Timo Sirainen date: Sat Sep 21 04:48:52 2013 +0300 description: lib-auth: Fixed infinite looping if user listing failed because auth process disconnected. diffstat: src/lib-auth/auth-master.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7a4ae8d511a2 -r 09f5e8d6b8d4 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Sat Sep 21 04:02:51 2013 +0300 +++ b/src/lib-auth/auth-master.c Sat Sep 21 04:48:52 2013 +0300 @@ -713,7 +713,7 @@ io_loop_set_current(ctx->conn->prev_ioloop); } - if (ctx->finished || ctx->failed) + if (ctx->finished || ctx->failed || ctx->conn->aborted) return NULL; return str_c(ctx->username); } From dovecot at dovecot.org Sat Sep 21 05:03:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 05:03:25 +0300 Subject: dovecot-2.2: auth: Fixed user iteration hang due to earlier stre... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b3634d4c362 changeset: 16797:8b3634d4c362 user: Timo Sirainen date: Sat Sep 21 05:03:03 2013 +0300 description: auth: Fixed user iteration hang due to earlier stream corking changes. Also fixed process title updating for auth-worker process during LIST. diffstat: src/auth/auth-master-connection.c | 6 +++- src/auth/auth-worker-client.c | 57 ++++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 19 deletions(-) diffs (167 lines): diff -r 09f5e8d6b8d4 -r 8b3634d4c362 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Sat Sep 21 04:48:52 2013 +0300 +++ b/src/auth/auth-master-connection.c Sat Sep 21 05:03:03 2013 +0300 @@ -447,8 +447,10 @@ master_input_list_finish(ctx); return 1; } - if (ret > 0) + if (ret > 0) { + o_stream_cork(ctx->conn->output); userdb_blocking_iter_next(ctx->iter); + } return 1; } @@ -502,6 +504,8 @@ } if (o_stream_get_buffer_used_size(ctx->conn->output) < MAX_OUTBUF_SIZE) userdb_blocking_iter_next(ctx->iter); + else + o_stream_uncork(ctx->conn->output); } static bool diff -r 09f5e8d6b8d4 -r 8b3634d4c362 src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Sat Sep 21 04:48:52 2013 +0300 +++ b/src/auth/auth-worker-client.c Sat Sep 21 05:03:03 2013 +0300 @@ -19,6 +19,7 @@ #define OUTBUF_THROTTLE_SIZE (1024*10) #define CLIENT_STATE_HANDSHAKE "handshaking" +#define CLIENT_STATE_ITER "iterating users" #define CLIENT_STATE_IDLE "idling" #define CLIENT_STATE_STOP "waiting for shutdown" @@ -413,6 +414,27 @@ return TRUE; } +static void +auth_worker_client_idle_kill(struct auth_worker_client *client ATTR_UNUSED) +{ + auth_worker_client_send_shutdown(); +} + +static void +auth_worker_client_set_idle_timeout(struct auth_worker_client *client) +{ + unsigned int idle_kill_secs; + + i_assert(client->to_idle == NULL); + + idle_kill_secs = master_service_get_idle_kill_secs(master_service); + if (idle_kill_secs > 0) { + client->to_idle = timeout_add(idle_kill_secs * 1000, + auth_worker_client_idle_kill, + client); + } +} + static void list_iter_deinit(struct auth_worker_list_context *ctx) { struct auth_worker_client *client = ctx->client; @@ -429,11 +451,14 @@ auth_worker_send_reply(client, str); client->io = io_add(client->fd, IO_READ, auth_worker_input, client); - o_stream_uncork(ctx->client->output); + auth_worker_client_set_idle_timeout(client); + o_stream_set_flush_callback(client->output, auth_worker_output, client); auth_request_unref(&ctx->auth_request); auth_worker_client_unref(&client); i_free(ctx); + + auth_worker_refresh_proctitle(CLIENT_STATE_IDLE); } static void list_iter_callback(const char *user, void *context) @@ -449,6 +474,8 @@ return; } + if (!ctx->sending) + o_stream_cork(ctx->client->output); T_BEGIN { str = t_str_new(128); str_printfa(str, "%u\t*\t%s\n", ctx->auth_request->id, user); @@ -476,9 +503,12 @@ } } while (ctx->sent && o_stream_get_buffer_used_size(ctx->client->output) <= OUTBUF_THROTTLE_SIZE); + o_stream_uncork(ctx->client->output); ctx->sending = FALSE; if (ctx->done) list_iter_deinit(ctx); + else + o_stream_set_flush_pending(ctx->client->output, TRUE); } static int auth_worker_list_output(struct auth_worker_list_context *ctx) @@ -528,7 +558,9 @@ } io_remove(&ctx->client->io); - o_stream_cork(ctx->client->output); + if (ctx->client->to_idle != NULL) + timeout_remove(&ctx->client->to_idle); + o_stream_set_flush_callback(ctx->client->output, auth_worker_list_output, ctx); ctx->iter = ctx->auth_request->userdb->userdb->iface-> @@ -566,7 +598,8 @@ i_error("BUG: Auth-worker received unknown command: %s", args[1]); } - auth_worker_refresh_proctitle(CLIENT_STATE_IDLE); + if (client->io != NULL) + auth_worker_refresh_proctitle(CLIENT_STATE_IDLE); return ret; } @@ -671,17 +704,10 @@ return 1; } -static void -auth_worker_client_idle_kill(struct auth_worker_client *client ATTR_UNUSED) -{ - auth_worker_client_send_shutdown(); -} - struct auth_worker_client * auth_worker_client_create(struct auth *auth, int fd) { struct auth_worker_client *client; - unsigned int idle_kill_secs; client = i_new(struct auth_worker_client, 1); client->refcount = 1; @@ -694,15 +720,9 @@ o_stream_set_no_error_handling(client->output, TRUE); o_stream_set_flush_callback(client->output, auth_worker_output, client); client->io = io_add(fd, IO_READ, auth_worker_input, client); + auth_worker_client_set_idle_timeout(client); auth_worker_refresh_proctitle(CLIENT_STATE_HANDSHAKE); - idle_kill_secs = master_service_get_idle_kill_secs(master_service); - if (idle_kill_secs > 0) { - client->to_idle = timeout_add(idle_kill_secs * 1000, - auth_worker_client_idle_kill, - client); - } - auth_worker_client = client; if (auth_worker_client_error) auth_worker_client_send_error(); @@ -767,7 +787,8 @@ o_stream_nsend_str(auth_worker_client->output, "SUCCESS\n"); auth_worker_client->error_sent = FALSE; } - auth_worker_refresh_proctitle(CLIENT_STATE_IDLE); + if (auth_worker_client->io != NULL) + auth_worker_refresh_proctitle(CLIENT_STATE_IDLE); } void auth_worker_client_send_shutdown(void) From dovecot at dovecot.org Sat Sep 21 05:05:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 05:05:37 +0300 Subject: dovecot-2.2: lib-storage: Don't create a settings cache for init... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2323c451150f changeset: 16798:2323c451150f user: Timo Sirainen date: Sat Sep 21 05:05:22 2013 +0300 description: lib-storage: Don't create a settings cache for initial global settings read. This fixes at least doveadm -A so that it doesn't connect to config process for each user. diffstat: src/lib-storage/mail-storage-service.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 8b3634d4c362 -r 2323c451150f src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Sat Sep 21 05:03:03 2013 +0300 +++ b/src/lib-storage/mail-storage-service.c Sat Sep 21 05:05:22 2013 +0300 @@ -864,7 +864,9 @@ set_input.local_ip = input->local_ip; set_input.remote_ip = input->remote_ip; } - if (ctx->set_cache == NULL) { + if (input == NULL) { + /* global settings read - don't create a cache for thi */ + } else if (ctx->set_cache == NULL) { ctx->set_cache_module = p_strdup(ctx->pool, set_input.module); ctx->set_cache_service = p_strdup(ctx->pool, set_input.service); ctx->set_cache = master_service_settings_cache_init( @@ -877,7 +879,8 @@ dyn_parsers = mail_storage_get_dynamic_parsers(pool); if (null_strcmp(set_input.module, ctx->set_cache_module) == 0 && - null_strcmp(set_input.service, ctx->set_cache_service) == 0) { + null_strcmp(set_input.service, ctx->set_cache_service) == 0 && + ctx->set_cache != NULL) { if (master_service_settings_cache_read(ctx->set_cache, &set_input, dyn_parsers, parser_r, error_r) < 0) { From dovecot at dovecot.org Sat Sep 21 05:28:55 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 05:28:55 +0300 Subject: dovecot-2.2: imap: Send * OK [CLOSED] always before tagged SELEC... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c019b298e0f6 changeset: 16799:c019b298e0f6 user: Timo Sirainen date: Sat Sep 21 05:28:45 2013 +0300 description: imap: Send * OK [CLOSED] always before tagged SELECT reply. diffstat: src/imap/cmd-select.c | 9 +++++---- src/imap/imap-commands-util.c | 25 +++++++++++++++++++------ src/imap/imap-commands-util.h | 3 +++ 3 files changed, 27 insertions(+), 10 deletions(-) diffs (109 lines): diff -r 2323c451150f -r c019b298e0f6 src/imap/cmd-select.c --- a/src/imap/cmd-select.c Sat Sep 21 05:05:22 2013 +0300 +++ b/src/imap/cmd-select.c Sat Sep 21 05:28:45 2013 +0300 @@ -394,7 +394,7 @@ struct client *client = cmd->client; struct imap_select_context *ctx; const struct imap_arg *args, *list_args; - const char *mailbox; + const char *mailbox, *error; int ret; /* [(optional parameters)] */ @@ -402,19 +402,20 @@ return FALSE; if (!imap_arg_get_astring(args, &mailbox)) { + close_selected_mailbox(client); client_send_command_error(cmd, "Invalid arguments."); - close_selected_mailbox(client); return FALSE; } ctx = p_new(cmd->pool, struct imap_select_context, 1); ctx->cmd = cmd; - ctx->ns = client_find_namespace(cmd, &mailbox); - (void)gettimeofday(&ctx->start_time, NULL); + ctx->ns = client_find_namespace_full(cmd->client, &mailbox, &error); if (ctx->ns == NULL) { close_selected_mailbox(client); + client_send_tagline(cmd, error); return TRUE; } + (void)gettimeofday(&ctx->start_time, NULL); if (imap_arg_get_list(&args[1], &list_args)) { if (!select_parse_options(ctx, list_args)) { diff -r 2323c451150f -r c019b298e0f6 src/imap/imap-commands-util.c --- a/src/imap/imap-commands-util.c Sat Sep 21 05:05:22 2013 +0300 +++ b/src/imap/imap-commands-util.c Sat Sep 21 05:28:45 2013 +0300 @@ -15,15 +15,16 @@ #include "imap-commands-util.h" struct mail_namespace * -client_find_namespace(struct client_command_context *cmd, const char **mailbox) +client_find_namespace_full(struct client *client, + const char **mailbox, const char **error_r) { - struct mail_namespace *namespaces = cmd->client->user->namespaces; + struct mail_namespace *namespaces = client->user->namespaces; struct mail_namespace *ns; string_t *utf8_name; utf8_name = t_str_new(64); if (imap_utf7_to_utf8(*mailbox, utf8_name) < 0) { - client_send_tagline(cmd, "NO Mailbox name is not valid mUTF-7"); + *error_r = "NO Mailbox name is not valid mUTF-7"; return NULL; } @@ -32,14 +33,14 @@ ns->prefix_len == 0) { /* this matched only the autocreated prefix="" namespace. give a nice human-readable error message */ - client_send_tagline(cmd, t_strdup_printf( + *error_r = t_strdup_printf( "NO Client tried to access nonexistent namespace. " "(Mailbox name should probably be prefixed with: %s)", - mail_namespace_find_inbox(namespaces)->prefix)); + mail_namespace_find_inbox(namespaces)->prefix); return NULL; } - if ((cmd->client->set->parsed_workarounds & + if ((client->set->parsed_workarounds & WORKAROUND_TB_EXTRA_MAILBOX_SEP) != 0 && str_len(utf8_name) > 0 && str_c(utf8_name)[str_len(utf8_name)-1] == mail_namespace_get_sep(ns)) { @@ -51,6 +52,18 @@ return ns; } +struct mail_namespace * +client_find_namespace(struct client_command_context *cmd, const char **mailbox) +{ + struct mail_namespace *ns; + const char *error; + + ns = client_find_namespace_full(cmd->client, mailbox, &error); + if (ns == NULL) + client_send_tagline(cmd, error); + return ns; +} + bool client_verify_open_mailbox(struct client_command_context *cmd) { if (cmd->client->mailbox != NULL) diff -r 2323c451150f -r c019b298e0f6 src/imap/imap-commands-util.h --- a/src/imap/imap-commands-util.h Sat Sep 21 05:05:22 2013 +0300 +++ b/src/imap/imap-commands-util.h Sat Sep 21 05:28:45 2013 +0300 @@ -13,6 +13,9 @@ or mailbox name is invalid, sends a tagged NO reply to client. */ struct mail_namespace * client_find_namespace(struct client_command_context *cmd, const char **mailbox); +struct mail_namespace * +client_find_namespace_full(struct client *client, + const char **mailbox, const char **error_r); /* Returns TRUE if mailbox is selected. If not, sends "No mailbox selected" error message to client. */ From dovecot at dovecot.org Sat Sep 21 23:35:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 21 Sep 2013 23:35:28 +0300 Subject: dovecot-2.2: lib-storage: Fixed listescape when escape_char was ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fce84463f508 changeset: 16800:fce84463f508 user: Timo Sirainen date: Sat Sep 21 23:33:42 2013 +0300 description: lib-storage: Fixed listescape when escape_char was the same as namespace separator. diffstat: src/lib-storage/mailbox-list.c | 12 ++++++++---- 1 files changed, 8 insertions(+), 4 deletions(-) diffs (43 lines): diff -r c019b298e0f6 -r fce84463f508 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Sat Sep 21 05:28:45 2013 +0300 +++ b/src/lib-storage/mailbox-list.c Sat Sep 21 23:33:42 2013 +0300 @@ -429,7 +429,7 @@ } for (; *vname != '\0'; vname++) { if (*vname == ns_sep) - str_append_c(escaped_name, *vname); + str_append_c(escaped_name, list_sep); else if (*vname == list_sep || *vname == list->set.escape_char || *vname == '/' || @@ -541,7 +541,7 @@ storage_name = "INBOX"; } - if (list_sep != ns_sep) { + if (list_sep != ns_sep && list->set.escape_char == '\0') { if (ns->type == MAIL_NAMESPACE_TYPE_SHARED && (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { /* shared namespace root. the backend storage's @@ -689,6 +689,12 @@ } prefix_len = strlen(list->ns->prefix); + if (list->set.escape_char != '\0') { + vname = mailbox_list_unescape_name(list, vname); + return prefix_len == 0 ? vname : + t_strconcat(list->ns->prefix, vname, NULL); + } + list_sep = mailbox_list_get_hierarchy_sep(list); ns_sep = mail_namespace_get_sep(list->ns); @@ -704,8 +710,6 @@ ret[i + prefix_len] = '\0'; vname = ret; } - if (list->set.escape_char != '\0') - vname = mailbox_list_unescape_name(list, vname); return vname; } From dovecot at dovecot.org Sun Sep 22 01:09:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 01:09:43 +0300 Subject: dovecot-2.2: dsync: Added -P parameter to do a purge for the rem... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/56be613e8ece changeset: 16801:56be613e8ece user: Timo Sirainen date: Sun Sep 22 01:09:32 2013 +0300 description: dsync: Added -P parameter to do a purge for the remote storage after syncing. diffstat: src/doveadm/dsync/doveadm-dsync.c | 10 ++++++++-- src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 26 ++++++++++++++++++++++++++ src/doveadm/dsync/dsync-brain.h | 5 ++++- src/doveadm/dsync/dsync-ibc-stream.c | 6 +++++- 5 files changed, 44 insertions(+), 4 deletions(-) diffs (176 lines): diff -r fce84463f508 -r 56be613e8ece src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sat Sep 21 23:33:42 2013 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Sun Sep 22 01:09:32 2013 +0300 @@ -36,7 +36,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:Nr:Rs:Ux:" +#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:" #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 /* The broken_char is mainly set to get a proper error message when trying to convert a mailbox with a name that can't be used properly translated between @@ -78,6 +78,7 @@ unsigned int lock_timeout; unsigned int lock:1; + unsigned int purge_remote:1; unsigned int sync_visible_namespaces:1; unsigned int default_replica_location:1; unsigned int oneway:1; @@ -364,6 +365,7 @@ } brain2 = dsync_brain_slave_init(user2, ibc2, TRUE); + mail_user_unref(&user2); brain1_running = brain2_running = TRUE; changed1 = changed2 = TRUE; @@ -376,7 +378,6 @@ brain1_running = dsync_brain_run(brain, &changed1); brain2_running = dsync_brain_run(brain2, &changed2); } - mail_user_unref(&user2); *changes_during_sync_r = dsync_brain_has_unexpected_changes(brain2); if (dsync_brain_deinit(&brain2) < 0) { ctx->ctx.exit_code = EX_TEMPFAIL; @@ -545,6 +546,8 @@ brain_flags = DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS; if (ctx->sync_visible_namespaces) brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; + if (ctx->purge_remote) + brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; if (ctx->reverse_backup) brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; @@ -904,6 +907,9 @@ case 'N': ctx->sync_visible_namespaces = TRUE; break; + case 'P': + ctx->purge_remote = TRUE; + break; case 'r': ctx->rawlog_path = optarg; break; diff -r fce84463f508 -r 56be613e8ece src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Sat Sep 21 23:33:42 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Sun Sep 22 01:09:32 2013 +0300 @@ -92,6 +92,7 @@ unsigned int mail_requests:1; unsigned int backup_send:1; unsigned int backup_recv:1; + unsigned int purge:1; unsigned int debug:1; unsigned int sync_visible_namespaces:1; unsigned int no_mail_sync:1; diff -r fce84463f508 -r 56be613e8ece src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Sat Sep 21 23:33:42 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Sun Sep 22 01:09:32 2013 +0300 @@ -104,6 +104,7 @@ pool_t pool; service_set = master_service_settings_get(master_service); + mail_user_ref(user); pool = pool_alloconly_create("dsync brain", 10240); brain = p_new(pool, struct dsync_brain, 1); @@ -220,6 +221,23 @@ return brain; } +static void dsync_brain_purge(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + struct mail_storage *storage; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + storage = mail_namespace_get_default_storage(ns); + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_error(storage, NULL)); + } + } +} + int dsync_brain_deinit(struct dsync_brain **_brain) { struct dsync_brain *brain = *_brain; @@ -240,6 +258,9 @@ brain->failed = TRUE; dsync_ibc_close_mail_streams(brain->ibc); + if (brain->purge && !brain->failed) + dsync_brain_purge(brain); + if (brain->box != NULL) dsync_brain_sync_mailbox_deinit(brain); if (brain->local_tree_iter != NULL) @@ -261,6 +282,7 @@ } ret = brain->failed ? -1 : 0; + mail_user_unref(&brain->user); pool_unref(&brain->pool); return ret; } @@ -383,7 +405,11 @@ sizeof(brain->sync_box_guid)); i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); brain->sync_type = ibc_set->sync_type; + dsync_brain_set_flags(brain, ibc_set->brain_flags); + /* this flag is only set on the remote slave brain */ + brain->purge = (ibc_set->brain_flags & + DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0; dsync_brain_mailbox_trees_init(brain); diff -r fce84463f508 -r 56be613e8ece src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Sat Sep 21 23:33:42 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Sun Sep 22 01:09:32 2013 +0300 @@ -19,7 +19,10 @@ /* Used with BACKUP_SEND/RECV: Don't force the Use the two-way syncing algorithm, but don't actually modify anything locally. (Useful during migration.) */ - DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40 + DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40, + /* Run storage purge on the remote after syncing. + Useful with e.g. a nightly doveadm backup. */ + DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80 }; enum dsync_brain_sync_type { diff -r fce84463f508 -r 56be613e8ece src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Sep 21 23:33:42 2013 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sun Sep 22 01:09:32 2013 +0300 @@ -75,7 +75,7 @@ .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type " "debug sync_visible_namespaces exclude_mailboxes " "send_mail_requests backup_send backup_recv lock_timeout " - "no_mail_sync no_backup_overwrite" + "no_mail_sync no_backup_overwrite purge_remote" }, { .name = "mailbox_state", .chr = 'S', @@ -660,6 +660,8 @@ dsync_serializer_encode_add(encoder, "no_mail_sync", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0) dsync_serializer_encode_add(encoder, "no_backup_overwrite", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0) + dsync_serializer_encode_add(encoder, "purge_remote", ""); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); @@ -750,6 +752,8 @@ set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; + if (dsync_deserializer_decode_try(decoder, "purge_remote", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; *set_r = set; return DSYNC_IBC_RECV_RET_OK; From dovecot at dovecot.org Sun Sep 22 01:40:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 01:40:42 +0300 Subject: dovecot-2.2: *-login: Added %{orig_user}, %{orig_username} and %... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a32eea97afc1 changeset: 16802:a32eea97afc1 user: Timo Sirainen date: Sun Sep 22 01:40:11 2013 +0300 description: *-login: Added %{orig_user}, %{orig_username} and %{orig_domain} variables. The original username is what the client sent to server before any translations. diffstat: src/auth/auth-request-handler.c | 5 +++++ src/login-common/client-common.c | 18 ++++++++++++++++++ src/login-common/client-common.h | 2 +- src/login-common/sasl-server.c | 9 +++++++++ 4 files changed, 33 insertions(+), 1 deletions(-) diffs (105 lines): diff -r 56be613e8ece -r a32eea97afc1 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Sun Sep 22 01:09:32 2013 +0300 +++ b/src/auth/auth-request-handler.c Sun Sep 22 01:40:11 2013 +0300 @@ -173,6 +173,11 @@ auth_fields_append(request->extra_fields, dest, AUTH_FIELD_FLAG_HIDDEN, 0); + if (strcmp(request->original_username, request->user) != 0) { + auth_str_add_keyvalue(dest, "original_user", + request->original_username); + } + if (!request->auth_only && auth_fields_exists(request->extra_fields, "proxy")) { /* we're proxying */ diff -r 56be613e8ece -r a32eea97afc1 src/login-common/client-common.c --- a/src/login-common/client-common.c Sun Sep 22 01:09:32 2013 +0300 +++ b/src/login-common/client-common.c Sun Sep 22 01:40:11 2013 +0300 @@ -272,6 +272,7 @@ i_free(client->proxy_user); i_free(client->proxy_master_user); i_free(client->virtual_user); + i_free(client->virtual_user_orig); i_free(client->auth_mech_name); i_free(client->master_data_prefix); pool_unref(&client->pool); @@ -453,6 +454,7 @@ { 'u', NULL, "user" }, { 'n', NULL, "username" }, { 'd', NULL, "domain" }, + { 's', NULL, "service" }, { 'h', NULL, "home" }, { 'l', NULL, "lip" }, @@ -469,6 +471,9 @@ { '\0', NULL, "real_rip" }, { '\0', NULL, "real_lport" }, { '\0', NULL, "real_rport" }, + { '\0', NULL, "orig_user" }, + { '\0', NULL, "orig_username" }, + { '\0', NULL, "orig_domain" }, { '\0', NULL, NULL } }; @@ -522,6 +527,19 @@ tab[16].value = net_ip2addr(&client->real_remote_ip); tab[17].value = dec2str(client->real_local_port); tab[18].value = dec2str(client->real_remote_port); + if (client->virtual_user_orig == NULL) { + tab[19].value = tab[0].value; + tab[20].value = tab[1].value; + tab[21].value = tab[2].value; + } else { + tab[19].value = client->virtual_user_orig; + tab[20].value = t_strcut(client->virtual_user_orig, '@'); + tab[21].value = strchr(client->virtual_user_orig, '@'); + if (tab[21].value != NULL) tab[21].value++; + + for (i = 0; i < 3; i++) + tab[i].value = str_sanitize(tab[i].value, 80); + } return tab; } diff -r 56be613e8ece -r a32eea97afc1 src/login-common/client-common.h --- a/src/login-common/client-common.h Sun Sep 22 01:09:32 2013 +0300 +++ b/src/login-common/client-common.h Sun Sep 22 01:40:11 2013 +0300 @@ -141,7 +141,7 @@ unsigned int auth_attempts, auth_successes; pid_t mail_pid; - char *virtual_user; + char *virtual_user, *virtual_user_orig; unsigned int destroyed:1; unsigned int input_blocked:1; unsigned int login_success:1; diff -r 56be613e8ece -r a32eea97afc1 src/login-common/sasl-server.c --- a/src/login-common/sasl-server.c Sun Sep 22 01:09:32 2013 +0300 +++ b/src/login-common/sasl-server.c Sun Sep 22 01:40:11 2013 +0300 @@ -239,7 +239,11 @@ for (i = 0; args[i] != NULL; i++) { if (strncmp(args[i], "user=", 5) == 0) { i_free(client->virtual_user); + i_free_and_null(client->virtual_user_orig); client->virtual_user = i_strdup(args[i] + 5); + } else if (strncmp(args[i], "original_user=", 14) == 0) { + i_free(client->virtual_user_orig); + client->virtual_user_orig = i_strdup(args[i] + 14); } else if (strcmp(args[i], "nologin") == 0 || strcmp(args[i], "proxy") == 0) { /* user can't login */ @@ -271,8 +275,13 @@ for (i = 0; args[i] != NULL; i++) { if (strncmp(args[i], "user=", 5) == 0) { i_free(client->virtual_user); + i_free_and_null(client->virtual_user_orig); client->virtual_user = i_strdup(args[i] + 5); + } else if (strncmp(args[i], "original_user=", 14) == 0) { + i_free(client->virtual_user_orig); + client->virtual_user_orig = + i_strdup(args[i] + 14); } } } From dovecot at dovecot.org Sun Sep 22 02:07:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:07:25 +0300 Subject: dovecot-2.2: mbox: Fixed mailbox_list_index=yes to work with non... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2a209302d064 changeset: 16803:2a209302d064 user: Timo Sirainen date: Sun Sep 22 02:07:16 2013 +0300 description: mbox: Fixed mailbox_list_index=yes to work with non-Dovecot mbox changes. diffstat: src/lib-storage/index/mbox/Makefile.am | 1 + src/lib-storage/index/mbox/mbox-storage.c | 5 +- src/lib-storage/index/mbox/mbox-storage.h | 8 +- src/lib-storage/index/mbox/mbox-sync-list-index.c | 93 +++++++++++++++++++++++ src/lib-storage/index/mbox/mbox-sync-private.h | 7 + 5 files changed, 111 insertions(+), 3 deletions(-) diffs (170 lines): diff -r a32eea97afc1 -r 2a209302d064 src/lib-storage/index/mbox/Makefile.am --- a/src/lib-storage/index/mbox/Makefile.am Sun Sep 22 01:40:11 2013 +0300 +++ b/src/lib-storage/index/mbox/Makefile.am Sun Sep 22 02:07:16 2013 +0300 @@ -19,6 +19,7 @@ mbox-md5-all.c \ mbox-save.c \ mbox-settings.c \ + mbox-sync-list-index.c \ mbox-sync-parse.c \ mbox-sync-rewrite.c \ mbox-sync-update.c \ diff -r a32eea97afc1 -r 2a209302d064 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 22 01:40:11 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 22 02:07:16 2013 +0300 @@ -379,6 +379,7 @@ mbox->storage = (struct mbox_storage *)storage; mbox->mbox_fd = -1; mbox->mbox_lock_type = F_UNLCK; + mbox->mbox_list_index_ext_id = (uint32_t)-1; if (strcmp(mbox->storage->set->mbox_md5, "apop3d") == 0) mbox->md5_v = mbox_md5_apop3d; @@ -836,8 +837,8 @@ index_storage_attribute_iter_init, index_storage_attribute_iter_next, index_storage_attribute_iter_deinit, - index_storage_list_index_has_changed, - index_storage_list_index_update_sync, + mbox_list_index_has_changed, + mbox_list_index_update_sync, mbox_storage_sync_init, index_mailbox_sync_next, index_mailbox_sync_deinit, diff -r a32eea97afc1 -r 2a209302d064 src/lib-storage/index/mbox/mbox-storage.h --- a/src/lib-storage/index/mbox/mbox-storage.h Sun Sep 22 01:40:11 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.h Sun Sep 22 02:07:16 2013 +0300 @@ -22,6 +22,12 @@ uint8_t unused[3]; guid_128_t mailbox_guid; }; + +struct mbox_list_index_record { + uint32_t mtime; + uint32_t size; +}; + struct mbox_storage { struct mail_storage storage; @@ -47,7 +53,7 @@ bool mbox_writeonly; unsigned int external_transactions; - uint32_t mbox_ext_idx, md5hdr_ext_idx; + uint32_t mbox_ext_idx, md5hdr_ext_idx, mbox_list_index_ext_id; struct mbox_index_header mbox_hdr; const struct mailbox_update *sync_hdr_update; diff -r a32eea97afc1 -r 2a209302d064 src/lib-storage/index/mbox/mbox-sync-list-index.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-list-index.c Sun Sep 22 02:07:16 2013 +0300 @@ -0,0 +1,93 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mbox-storage.h" +#include "mbox-sync-private.h" + +static unsigned int +mbox_list_get_ext_id(struct mbox_mailbox *mbox, + struct mail_index_view *view) +{ + if (mbox->mbox_list_index_ext_id == (uint32_t)-1) { + mbox->mbox_list_index_ext_id = + mail_index_ext_register(mail_index_view_get_index(view), + "mbox", 0, + sizeof(struct mbox_list_index_record), + sizeof(uint32_t)); + } + return mbox->mbox_list_index_ext_id; +} + +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq) +{ + struct mbox_mailbox *mbox = (struct mbox_mailbox *)box; + const struct mbox_list_index_record *rec; + const void *data; + const char *path; + struct stat st; + uint32_t ext_id; + bool expunged; + int ret; + + ret = index_storage_list_index_has_changed(box, list_view, seq); + if (ret != 0) + return ret; + + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + rec = data; + + if (rec == NULL || expunged || rec->mtime == 0) { + /* doesn't exist or not synced */ + return 1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, &path); + if (ret < 0) + return ret; + i_assert(ret > 0); + + if (stat(path, &st) < 0) { + mail_storage_set_critical(box->storage, + "stat(%s) failed: %m", path); + return -1; + } + if ((time_t)rec->mtime != st.st_mtime || + rec->size != (uint32_t)(st.st_size & 0xffffffffU)) + return 1; + return 0; +} + +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq) +{ + struct mbox_mailbox *mbox = (struct mbox_mailbox *)box; + struct mail_index_view *list_view; + const struct mbox_index_header *mhdr = &mbox->mbox_hdr; + const struct mbox_list_index_record *old_rec; + struct mbox_list_index_record new_rec; + const void *data; + uint32_t ext_id; + bool expunged; + + index_storage_list_index_update_sync(box, trans, seq); + + /* get the current record */ + list_view = mail_index_transaction_get_view(trans); + ext_id = mbox_list_get_ext_id(mbox, list_view); + mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); + if (expunged) + return; + old_rec = data; + + memset(&new_rec, 0, sizeof(new_rec)); + new_rec.mtime = mhdr->sync_mtime; + new_rec.size = mhdr->sync_size & 0xffffffffU; + + if (old_rec == NULL || + memcmp(old_rec, &new_rec, sizeof(*old_rec)) != 0) + mail_index_update_ext(trans, seq, ext_id, &new_rec, NULL); +} diff -r a32eea97afc1 -r 2a209302d064 src/lib-storage/index/mbox/mbox-sync-private.h --- a/src/lib-storage/index/mbox/mbox-sync-private.h Sun Sep 22 01:40:11 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-private.h Sun Sep 22 02:07:16 2013 +0300 @@ -183,4 +183,11 @@ size_t size); int mbox_sync_get_guid(struct mbox_mailbox *mbox); +int mbox_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq); +void mbox_list_index_update_sync(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t seq); + #endif From dovecot at dovecot.org Sun Sep 22 02:20:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:20:16 +0300 Subject: dovecot-2.2: Added ssl_prefer_server_ciphers setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/897484f45a87 changeset: 16804:897484f45a87 user: Timo Sirainen date: Sun Sep 22 02:20:09 2013 +0300 description: Added ssl_prefer_server_ciphers setting. diffstat: doc/example-config/conf.d/10-ssl.conf | 3 +++ src/lib-master/master-service-ssl-settings.c | 4 +++- src/lib-master/master-service-ssl-settings.h | 1 + src/lib-master/master-service-ssl.c | 1 + src/lib-ssl-iostream/iostream-openssl-context.c | 4 ++++ src/lib-ssl-iostream/iostream-openssl.c | 2 ++ src/lib-ssl-iostream/iostream-ssl.h | 1 + src/login-common/ssl-proxy-openssl.c | 5 +++++ 8 files changed, 20 insertions(+), 1 deletions(-) diffs (128 lines): diff -r 2a209302d064 -r 897484f45a87 doc/example-config/conf.d/10-ssl.conf --- a/doc/example-config/conf.d/10-ssl.conf Sun Sep 22 02:07:16 2013 +0300 +++ b/doc/example-config/conf.d/10-ssl.conf Sun Sep 22 02:20:09 2013 +0300 @@ -53,5 +53,8 @@ # SSL ciphers to use #ssl_cipher_list = ALL:!LOW:!SSLv2:!EXP:!aNULL +# Prefer the server's order of ciphers over client's. +#ssl_prefer_server_ciphers = no + # SSL crypto device to use, for valid values run "openssl engine" #ssl_crypto_device = diff -r 2a209302d064 -r 897484f45a87 src/lib-master/master-service-ssl-settings.c --- a/src/lib-master/master-service-ssl-settings.c Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-master/master-service-ssl-settings.c Sun Sep 22 02:20:09 2013 +0300 @@ -27,6 +27,7 @@ DEF(SET_BOOL, ssl_verify_client_cert), DEF(SET_BOOL, ssl_require_crl), DEF(SET_BOOL, verbose_ssl), + DEF(SET_BOOL, ssl_prefer_server_ciphers), SETTING_DEFINE_LIST_END }; @@ -47,7 +48,8 @@ .ssl_crypto_device = "", .ssl_verify_client_cert = FALSE, .ssl_require_crl = TRUE, - .verbose_ssl = FALSE + .verbose_ssl = FALSE, + .ssl_prefer_server_ciphers = FALSE }; const struct setting_parser_info master_service_ssl_setting_parser_info = { diff -r 2a209302d064 -r 897484f45a87 src/lib-master/master-service-ssl-settings.h --- a/src/lib-master/master-service-ssl-settings.h Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-master/master-service-ssl-settings.h Sun Sep 22 02:20:09 2013 +0300 @@ -16,6 +16,7 @@ bool ssl_verify_client_cert; bool ssl_require_crl; bool verbose_ssl; + bool ssl_prefer_server_ciphers; }; extern const struct setting_parser_info master_service_ssl_setting_parser_info; diff -r 2a209302d064 -r 897484f45a87 src/lib-master/master-service-ssl.c --- a/src/lib-master/master-service-ssl.c Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-master/master-service-ssl.c Sun Sep 22 02:20:09 2013 +0300 @@ -119,6 +119,7 @@ ssl_set.verbose = set->verbose_ssl; ssl_set.verify_remote_cert = set->ssl_verify_client_cert; + ssl_set.prefer_server_ciphers = set->ssl_prefer_server_ciphers; if (ssl_iostream_context_init_server(&ssl_set, &service->ssl_ctx, &error) < 0) { diff -r 2a209302d064 -r 897484f45a87 src/lib-ssl-iostream/iostream-openssl-context.c --- a/src/lib-ssl-iostream/iostream-openssl-context.c Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl-context.c Sun Sep 22 02:20:09 2013 +0300 @@ -369,6 +369,10 @@ set->cipher_list, openssl_iostream_error()); return -1; } + if (set->prefer_server_ciphers) { + SSL_CTX_set_options(ctx->ssl_ctx, + SSL_OP_CIPHER_SERVER_PREFERENCE); + } if (ctx->set->protocols != NULL) { SSL_CTX_set_options(ctx->ssl_ctx, openssl_get_protocol_options(ctx->set->protocols)); diff -r 2a209302d064 -r 897484f45a87 src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.c Sun Sep 22 02:20:09 2013 +0300 @@ -154,6 +154,8 @@ return -1; } } + if (set->prefer_server_ciphers) + SSL_set_options(ssl_io->ssl, SSL_OP_CIPHER_SERVER_PREFERENCE); if (set->protocols != NULL) { SSL_clear_options(ssl_io->ssl, OPENSSL_ALL_PROTOCOL_OPTIONS); SSL_set_options(ssl_io->ssl, diff -r 2a209302d064 -r 897484f45a87 src/lib-ssl-iostream/iostream-ssl.h --- a/src/lib-ssl-iostream/iostream-ssl.h Sun Sep 22 02:07:16 2013 +0300 +++ b/src/lib-ssl-iostream/iostream-ssl.h Sun Sep 22 02:20:09 2013 +0300 @@ -17,6 +17,7 @@ bool verbose, verbose_invalid_cert; /* stream-only */ bool verify_remote_cert; /* neither/both */ bool require_valid_cert; /* stream-only */ + bool prefer_server_ciphers; }; /* Returns 0 if ok, -1 and sets error_r if failed. The returned error string diff -r 2a209302d064 -r 897484f45a87 src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Sun Sep 22 02:07:16 2013 +0300 +++ b/src/login-common/ssl-proxy-openssl.c Sun Sep 22 02:20:09 2013 +0300 @@ -99,6 +99,7 @@ const char *cipher_list; const char *protocols; bool verify_client_cert; + bool prefer_server_ciphers; }; static int extdata_index; @@ -634,6 +635,7 @@ lookup_ctx.verify_client_cert = set->ssl_verify_client_cert || login_set->auth_ssl_require_client_cert || login_set->auth_ssl_username_from_cert; + lookup_ctx.prefer_server_ciphers = set->ssl_prefer_server_ciphers; ctx = hash_table_lookup(ssl_servers, &lookup_ctx); if (ctx == NULL) @@ -1271,6 +1273,7 @@ ctx->verify_client_cert = ssl_set->ssl_verify_client_cert || login_set->auth_ssl_require_client_cert || login_set->auth_ssl_username_from_cert; + ctx->prefer_server_ciphers = ssl_set->ssl_prefer_server_ciphers; ctx->ctx = ssl_ctx = SSL_CTX_new(SSLv23_server_method()); if (ssl_ctx == NULL) @@ -1281,6 +1284,8 @@ i_fatal("Can't set cipher list to '%s': %s", ctx->cipher_list, ssl_last_error()); } + if (ctx->prefer_server_ciphers) + SSL_CTX_set_options(ssl_ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); SSL_CTX_set_options(ssl_ctx, openssl_get_protocol_options(ctx->protocols)); if (ssl_proxy_ctx_use_certificate_chain(ctx->ctx, ctx->cert) != 1) { From dovecot at dovecot.org Sun Sep 22 02:24:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:24:19 +0300 Subject: dovecot-2.2: If EOVERFLOW errno isn't defined by system, fallbac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d064c57a601d changeset: 16805:d064c57a601d user: Timo Sirainen date: Sun Sep 22 02:23:30 2013 +0300 description: If EOVERFLOW errno isn't defined by system, fallback to ERANGE instead of EINVAL. ERANGE is used by other functions as well, such as getpwnam_r(). diffstat: src/lib/compat.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 897484f45a87 -r d064c57a601d src/lib/compat.h --- a/src/lib/compat.h Sun Sep 22 02:20:09 2013 +0300 +++ b/src/lib/compat.h Sun Sep 22 02:23:30 2013 +0300 @@ -226,7 +226,7 @@ #define i_isxdigit(x) isxdigit((int) (unsigned char) (x)) #ifndef EOVERFLOW -# define EOVERFLOW EINVAL +# define EOVERFLOW ERANGE #endif #ifdef EDQUOT From dovecot at dovecot.org Sun Sep 22 02:24:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:24:19 +0300 Subject: dovecot-2.2: net_listen_unix(): If path is too long, return EOVE... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07cc79e0041d changeset: 16806:07cc79e0041d user: Timo Sirainen date: Sun Sep 22 02:24:05 2013 +0300 description: net_listen_unix(): If path is too long, return EOVERFLOW instead of EINVAL. diffstat: src/lib/net.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d064c57a601d -r 07cc79e0041d src/lib/net.c --- a/src/lib/net.c Sun Sep 22 02:23:30 2013 +0300 +++ b/src/lib/net.c Sun Sep 22 02:24:05 2013 +0300 @@ -461,7 +461,7 @@ sa.un.sun_family = AF_UNIX; if (i_strocpy(sa.un.sun_path, path, sizeof(sa.un.sun_path)) < 0) { /* too long path */ - errno = EINVAL; + errno = EOVERFLOW; return -1; } From dovecot at dovecot.org Sun Sep 22 02:32:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:32:17 +0300 Subject: dovecot-2.2: auth: Use a common auth_request_log_unknown_user() ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4ce8f47d20af changeset: 16807:4ce8f47d20af user: Timo Sirainen date: Sun Sep 22 02:32:05 2013 +0300 description: auth: Use a common auth_request_log_unknown_user() for logging "unknown user" entries. diffstat: src/auth/auth-request.c | 6 ++++++ src/auth/auth-request.h | 2 ++ src/auth/db-passwd-file.c | 2 +- src/auth/passdb-bsdauth.c | 2 +- src/auth/passdb-cache.c | 2 +- src/auth/passdb-dict.c | 2 +- src/auth/passdb-ldap.c | 9 +++------ src/auth/passdb-pam.c | 5 ++++- src/auth/passdb-passwd.c | 2 +- src/auth/passdb-shadow.c | 2 +- src/auth/passdb-sql.c | 2 +- src/auth/userdb-dict.c | 2 +- src/auth/userdb-ldap.c | 3 +-- src/auth/userdb-nss.c | 2 +- src/auth/userdb-passwd.c | 2 +- src/auth/userdb-sql.c | 2 +- src/auth/userdb-vpopmail.c | 2 +- 17 files changed, 28 insertions(+), 21 deletions(-) diffs (233 lines): diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/auth-request.c --- a/src/auth/auth-request.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/auth-request.c Sun Sep 22 02:32:05 2013 +0300 @@ -2114,6 +2114,12 @@ va_end(va); } +void auth_request_log_unknown_user(struct auth_request *auth_request, + const char *subsystem) +{ + auth_request_log_info(auth_request, subsystem, "unknown user"); +} + void auth_request_refresh_last_access(struct auth_request *request) { request->last_access = ioloop_time; diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/auth-request.h --- a/src/auth/auth-request.h Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/auth-request.h Sun Sep 22 02:32:05 2013 +0300 @@ -245,6 +245,8 @@ void auth_request_log_error(struct auth_request *auth_request, const char *subsystem, const char *format, ...) ATTR_FORMAT(3, 4); +void auth_request_log_unknown_user(struct auth_request *auth_request, + const char *subsystem); void auth_request_verify_plain_callback(enum passdb_result result, struct auth_request *request); diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/db-passwd-file.c --- a/src/auth/db-passwd-file.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/db-passwd-file.c Sun Sep 22 02:32:05 2013 +0300 @@ -463,7 +463,7 @@ pu = hash_table_lookup(pw->users, str_c(username)); if (pu == NULL) - auth_request_log_info(request, "passwd-file", "unknown user"); + auth_request_log_unknown_user(request, "passwd-file"); return pu; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-bsdauth.c --- a/src/auth/passdb-bsdauth.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-bsdauth.c Sun Sep 22 02:32:05 2013 +0300 @@ -30,7 +30,7 @@ callback(PASSDB_RESULT_INTERNAL_FAILURE, request); return; case 0: - auth_request_log_info(request, "bsdauth", "unknown user"); + auth_request_log_unknown_user(request, "bsdauth"); callback(PASSDB_RESULT_USER_UNKNOWN, request); return; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-cache.c --- a/src/auth/passdb-cache.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-cache.c Sun Sep 22 02:32:05 2013 +0300 @@ -48,7 +48,7 @@ if (*value == '\0') { /* negative cache entry */ - auth_request_log_info(request, "cache", "User unknown"); + auth_request_log_unknown_user(request, "cache"); *result_r = PASSDB_RESULT_USER_UNKNOWN; return TRUE; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-dict.c --- a/src/auth/passdb-dict.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-dict.c Sun Sep 22 02:32:05 2013 +0300 @@ -64,7 +64,7 @@ auth_request_log_error(auth_request, "dict", "Lookup failed"); return PASSDB_RESULT_INTERNAL_FAILURE; } else if (ret == 0) { - auth_request_log_info(auth_request, "dict", "unknown user"); + auth_request_log_unknown_user(auth_request, "dict"); return PASSDB_RESULT_USER_UNKNOWN; } else { auth_request_log_debug(auth_request, "dict", diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-ldap.c --- a/src/auth/passdb-ldap.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-ldap.c Sun Sep 22 02:32:05 2013 +0300 @@ -77,8 +77,7 @@ passdb_result = PASSDB_RESULT_INTERNAL_FAILURE; } else if (ldap_request->entries == 0) { passdb_result = PASSDB_RESULT_USER_UNKNOWN; - auth_request_log_info(auth_request, "ldap", - "unknown user"); + auth_request_log_unknown_user(auth_request, "ldap"); } else if (ldap_request->entries > 1) { auth_request_log_error(auth_request, "ldap", "pass_filter matched multiple objects, aborting"); @@ -166,8 +165,7 @@ passdb_result = PASSDB_RESULT_PASSWORD_MISMATCH; } else if (ret == LDAP_NO_SUCH_OBJECT) { passdb_result = PASSDB_RESULT_USER_UNKNOWN; - auth_request_log_info(auth_request, "ldap", - "unknown user"); + auth_request_log_unknown_user(auth_request, "ldap"); } else { auth_request_log_error(auth_request, "ldap", "ldap_bind() failed: %s", @@ -214,8 +212,7 @@ passdb_result = PASSDB_RESULT_INTERNAL_FAILURE; else if (request->entries == 0) { passdb_result = PASSDB_RESULT_USER_UNKNOWN; - auth_request_log_info(auth_request, "ldap", - "unknown user"); + auth_request_log_unknown_user(auth_request, "ldap"); } else { i_assert(request->entries > 1); auth_request_log_error(auth_request, "ldap", diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-pam.c --- a/src/auth/passdb-pam.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-pam.c Sun Sep 22 02:32:05 2013 +0300 @@ -181,7 +181,10 @@ } auth_request_log_info(request, "pam", "%s", str); } else { - auth_request_log_info(request, "pam", "%s", str); + if (status == PAM_USER_UNKNOWN) + auth_request_log_unknown_user(request, "pam"); + else + auth_request_log_info(request, "pam", "%s", str); } return status; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-passwd.c --- a/src/auth/passdb-passwd.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-passwd.c Sun Sep 22 02:32:05 2013 +0300 @@ -27,7 +27,7 @@ callback(PASSDB_RESULT_INTERNAL_FAILURE, request); return; case 0: - auth_request_log_info(request, "passwd", "unknown user"); + auth_request_log_unknown_user(request, "passwd"); callback(PASSDB_RESULT_USER_UNKNOWN, request); return; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-shadow.c --- a/src/auth/passdb-shadow.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-shadow.c Sun Sep 22 02:32:05 2013 +0300 @@ -23,7 +23,7 @@ spw = getspnam(request->user); if (spw == NULL) { - auth_request_log_info(request, "shadow", "unknown user"); + auth_request_log_unknown_user(request, "shadow"); callback(PASSDB_RESULT_USER_UNKNOWN, request); return; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/passdb-sql.c --- a/src/auth/passdb-sql.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/passdb-sql.c Sun Sep 22 02:32:05 2013 +0300 @@ -85,7 +85,7 @@ module->conn->set.password_query); } } else if (ret == 0) { - auth_request_log_info(auth_request, "sql", "unknown user"); + auth_request_log_unknown_user(auth_request, "sql"); passdb_result = PASSDB_RESULT_USER_UNKNOWN; } else { sql_query_save_results(result, sql_request); diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-dict.c --- a/src/auth/userdb-dict.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-dict.c Sun Sep 22 02:32:05 2013 +0300 @@ -81,7 +81,7 @@ auth_request_log_error(auth_request, "dict", "Lookup failed"); userdb_result = USERDB_RESULT_INTERNAL_FAILURE; } else if (ret == 0) { - auth_request_log_info(auth_request, "dict", "unknown user"); + auth_request_log_unknown_user(auth_request, "dict"); userdb_result = USERDB_RESULT_USER_UNKNOWN; } else { auth_request_log_debug(auth_request, "dict", diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-ldap.c --- a/src/auth/userdb-ldap.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-ldap.c Sun Sep 22 02:32:05 2013 +0300 @@ -71,8 +71,7 @@ result = USERDB_RESULT_INTERNAL_FAILURE; } else if (urequest->entries == 0) { result = USERDB_RESULT_USER_UNKNOWN; - auth_request_log_info(auth_request, "ldap", - "unknown user"); + auth_request_log_unknown_user(auth_request, "ldap"); } else if (urequest->entries > 1) { auth_request_log_error(auth_request, "ldap", "user_filter matched multiple objects, aborting"); diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-nss.c --- a/src/auth/userdb-nss.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-nss.c Sun Sep 22 02:32:05 2013 +0300 @@ -52,7 +52,7 @@ "unavailable (err=%d)", err); break; case NSS_STATUS_NOTFOUND: - auth_request_log_info(auth_request, "nss", "unknown user"); + auth_request_log_unknown_user(auth_request, "nss"); result = USERDB_RESULT_USER_UNKNOWN; break; case NSS_STATUS_SUCCESS: diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-passwd.c Sun Sep 22 02:32:05 2013 +0300 @@ -101,7 +101,7 @@ callback(USERDB_RESULT_INTERNAL_FAILURE, auth_request); return; case 0: - auth_request_log_info(auth_request, "passwd", "unknown user"); + auth_request_log_unknown_user(auth_request, "passwd"); callback(USERDB_RESULT_USER_UNKNOWN, auth_request); return; } diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-sql.c --- a/src/auth/userdb-sql.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-sql.c Sun Sep 22 02:32:05 2013 +0300 @@ -83,7 +83,7 @@ } } else if (ret == 0) { result = USERDB_RESULT_USER_UNKNOWN; - auth_request_log_info(auth_request, "sql", "Unknown user"); + auth_request_log_unknown_user(auth_request, "sql"); } else { sql_query_get_result(sql_result, auth_request); result = USERDB_RESULT_OK; diff -r 07cc79e0041d -r 4ce8f47d20af src/auth/userdb-vpopmail.c --- a/src/auth/userdb-vpopmail.c Sun Sep 22 02:24:05 2013 +0300 +++ b/src/auth/userdb-vpopmail.c Sun Sep 22 02:32:05 2013 +0300 @@ -42,7 +42,7 @@ vpw = vauth_getpw(vpop_user, vpop_domain); if (vpw == NULL) { - auth_request_log_info(request, "vpopmail", "unknown user"); + auth_request_log_unknown_user(request, "vpopmail"); return NULL; } From dovecot at dovecot.org Sun Sep 22 02:44:50 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 02:44:50 +0300 Subject: dovecot-2.2: auth: If auth_verbose_passwords is set, log the pas... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1f9294fbb118 changeset: 16808:1f9294fbb118 user: Timo Sirainen date: Sun Sep 22 02:44:21 2013 +0300 description: auth: If auth_verbose_passwords is set, log the password also for unknown users. diffstat: src/auth/auth-request.c | 54 ++++++++++++++++++++++++++++++++---------------- 1 files changed, 36 insertions(+), 18 deletions(-) diffs (82 lines): diff -r 4ce8f47d20af -r 1f9294fbb118 src/auth/auth-request.c --- a/src/auth/auth-request.c Sun Sep 22 02:32:05 2013 +0300 +++ b/src/auth/auth-request.c Sun Sep 22 02:44:21 2013 +0300 @@ -1782,21 +1782,11 @@ auth_request_log_debug(request, subsystem, "%s", str_c(str)); } -void auth_request_log_password_mismatch(struct auth_request *request, - const char *subsystem) +static void +auth_request_append_password(struct auth_request *request, string_t *str) { - string_t *str; const char *log_type = request->set->verbose_passwords; - if (strcmp(log_type, "no") == 0) { - auth_request_log_info(request, subsystem, "Password mismatch"); - return; - } - - str = t_str_new(128); - get_log_prefix(str, request, subsystem); - str_append(str, "Password mismatch "); - if (strcmp(log_type, "plain") == 0) { str_printfa(str, "(given password: %s)", request->mech_password); @@ -1810,7 +1800,41 @@ } else { i_unreached(); } +} +void auth_request_log_password_mismatch(struct auth_request *request, + const char *subsystem) +{ + string_t *str; + + if (strcmp(request->set->verbose_passwords, "no") == 0) { + auth_request_log_info(request, subsystem, "Password mismatch"); + return; + } + + str = t_str_new(128); + get_log_prefix(str, request, subsystem); + str_append(str, "Password mismatch "); + + auth_request_append_password(request, str); + i_info("%s", str_c(str)); +} + +void auth_request_log_unknown_user(struct auth_request *request, + const char *subsystem) +{ + string_t *str; + + if (strcmp(request->set->verbose_passwords, "no") == 0 || + !request->set->verbose) { + auth_request_log_info(request, subsystem, "unknown user"); + return; + } + str = t_str_new(128); + get_log_prefix(str, request, subsystem); + str_append(str, "unknown user "); + + auth_request_append_password(request, str); i_info("%s", str_c(str)); } @@ -2114,12 +2138,6 @@ va_end(va); } -void auth_request_log_unknown_user(struct auth_request *auth_request, - const char *subsystem) -{ - auth_request_log_info(auth_request, subsystem, "unknown user"); -} - void auth_request_refresh_last_access(struct auth_request *request) { request->last_access = ioloop_time; From dovecot at dovecot.org Sun Sep 22 03:40:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 03:40:27 +0300 Subject: dovecot-2.2: example-config: Fixed login_log_format comment. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d400c1a673cf changeset: 16809:d400c1a673cf user: Timo Sirainen date: Sun Sep 22 03:17:12 2013 +0300 description: example-config: Fixed login_log_format comment. diffstat: doc/example-config/conf.d/10-logging.conf | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1f9294fbb118 -r d400c1a673cf doc/example-config/conf.d/10-logging.conf --- a/doc/example-config/conf.d/10-logging.conf Sun Sep 22 02:44:21 2013 +0300 +++ b/doc/example-config/conf.d/10-logging.conf Sun Sep 22 03:17:12 2013 +0300 @@ -65,7 +65,7 @@ # string. #login_log_format_elements = user=<%u> method=%m rip=%r lip=%l mpid=%e %c -# Login log format. %$ contains login_log_format_elements string, %s contains +# Login log format. %s contains login_log_format_elements string, %$ contains # the data we want to log. #login_log_format = %$: %s From dovecot at dovecot.org Sun Sep 22 03:40:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 03:40:27 +0300 Subject: dovecot-2.2: acl: Hide non-listable mailboxes from a shared subs... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1cf67db75455 changeset: 16810:1cf67db75455 user: Timo Sirainen date: Sun Sep 22 03:40:14 2013 +0300 description: acl: Hide non-listable mailboxes from a shared subscriptions file. diffstat: src/plugins/acl/acl-mailbox-list.c | 31 +++++++++++++++++++++++++++---- 1 files changed, 27 insertions(+), 4 deletions(-) diffs (87 lines): diff -r d400c1a673cf -r 1cf67db75455 src/plugins/acl/acl-mailbox-list.c --- a/src/plugins/acl/acl-mailbox-list.c Sun Sep 22 03:17:12 2013 +0300 +++ b/src/plugins/acl/acl-mailbox-list.c Sun Sep 22 03:40:14 2013 +0300 @@ -22,6 +22,7 @@ struct mailbox_info info; char sep; + unsigned int hide_nonlistable_subscriptions:1; unsigned int simple_star_glob:1; }; @@ -172,6 +173,14 @@ ctx->ctx.list = list; ctx->ctx.flags = flags; + if (list->ns->type != MAIL_NAMESPACE_TYPE_PRIVATE && + (list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0) { + /* non-private namespace with subscriptions=yes. this could be + a site-global subscriptions file, so hide subscriptions for + mailboxes the user doesn't see. */ + ctx->hide_nonlistable_subscriptions = TRUE; + } + inboxcase = (list->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0; ctx->sep = mail_namespace_get_sep(list->ns); ctx->ctx.glob = imap_match_init_multiple(pool, patterns, @@ -253,7 +262,7 @@ static bool iter_mailbox_has_visible_children(struct acl_mailbox_list_iterate_context *ctx, - bool only_nonpatterns) + bool only_nonpatterns, bool subscribed) { struct mailbox_list_iterate_context *iter; const struct mailbox_info *info; @@ -295,6 +304,8 @@ prefix_len = str_len(pattern) - 1; iter = mailbox_list_iter_init(ctx->ctx.list, str_c(pattern), + (!subscribed ? 0 : + MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) | MAILBOX_LIST_ITER_RETURN_NO_FLAGS); while ((info = mailbox_list_iter_next(iter)) != NULL) { if (only_nonpatterns && @@ -325,7 +336,8 @@ } if ((ctx->ctx.flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0 && - (ctx->ctx.flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) != 0) { + (ctx->ctx.flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) != 0 && + !ctx->hide_nonlistable_subscriptions) { /* don't waste time doing an ACL check. we're going to list all subscriptions anyway. */ info->flags &= MAILBOX_SUBSCRIBED | MAILBOX_CHILD_SUBSCRIBED; @@ -342,7 +354,7 @@ children, but also don't return incorrect flags */ info->flags &= ~MAILBOX_CHILDREN; } else if ((info->flags & MAILBOX_CHILDREN) != 0 && - !iter_mailbox_has_visible_children(ctx, FALSE)) { + !iter_mailbox_has_visible_children(ctx, FALSE, FALSE)) { info->flags &= ~MAILBOX_CHILDREN; info->flags |= MAILBOX_NOCHILDREN; } @@ -357,11 +369,22 @@ i_assert((info->flags & PRESERVE_MAILBOX_FLAGS) != 0); info->flags = MAILBOX_NONEXISTENT | (info->flags & PRESERVE_MAILBOX_FLAGS); + if (ctx->hide_nonlistable_subscriptions) { + /* global subscriptions file. hide this entry if there + are no visible subscribed children or if we're going + to list the subscribed children anyway. */ + if ((info->flags & MAILBOX_CHILD_SUBSCRIBED) == 0) + return 0; + if (iter_is_listing_all_children(ctx) || + !iter_mailbox_has_visible_children(ctx, TRUE, TRUE)) + return 0; + /* e.g. LSUB "" % with visible subscribed children */ + } return 1; } if (!iter_is_listing_all_children(ctx) && - iter_mailbox_has_visible_children(ctx, TRUE)) { + iter_mailbox_has_visible_children(ctx, TRUE, FALSE)) { /* no child mailboxes match the list pattern(s), but mailbox has visible children. we'll need to show this as non-existent. */ From dovecot at dovecot.org Sun Sep 22 04:14:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 04:14:51 +0300 Subject: dovecot-2.2: Fixes and improvements to istream-attachment-extrac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4f68ac02f46c changeset: 16811:4f68ac02f46c user: Timo Sirainen date: Sun Sep 22 04:14:23 2013 +0300 description: Fixes and improvements to istream-attachment-extractor error handling. If an attachment saving failed, the mail was still saved to disk, just without the attachments. diffstat: src/lib-mail/istream-attachment-extractor.c | 38 +++++++++++++++++++--------- src/lib-mail/istream-attachment-extractor.h | 4 +- src/lib-mail/test-istream-attachment.c | 2 + src/lib-storage/index/index-attachment.c | 22 +++++++++++----- 4 files changed, 45 insertions(+), 21 deletions(-) diffs (233 lines): diff -r 1cf67db75455 -r 4f68ac02f46c src/lib-mail/istream-attachment-extractor.c --- a/src/lib-mail/istream-attachment-extractor.c Sun Sep 22 03:40:14 2013 +0300 +++ b/src/lib-mail/istream-attachment-extractor.c Sun Sep 22 04:14:23 2013 +0300 @@ -59,6 +59,7 @@ struct attachment_istream_part part; bool retry_read; + bool failed; }; static void stream_add_data(struct attachment_istream *astream, @@ -416,7 +417,8 @@ return 0; } -static int astream_part_finish(struct attachment_istream *astream) +static int +astream_part_finish(struct attachment_istream *astream, const char **error_r) { struct attachment_istream_part *part = &astream->part; struct istream_attachment_info info; @@ -428,8 +430,9 @@ int ret = 0; if (o_stream_nfinish(part->temp_output) < 0) { - i_error("istream-attachment: write(%s) failed: %m", - o_stream_get_name(part->temp_output)); + *error_r = t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(part->temp_output), + o_stream_get_error(part->temp_output)); return -1; } @@ -477,7 +480,7 @@ as attachment */ info.encoded_size = part->temp_output->offset; } - if (astream->set.open_attachment_ostream(&info, &output, + if (astream->set.open_attachment_ostream(&info, &output, error_r, astream->context) < 0) return -1; @@ -489,13 +492,13 @@ } if (input->stream_errno != 0) { - i_error("istream-attachment: read(%s) failed: %m", - i_stream_get_name(input)); + *error_r = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); ret = -1; } i_stream_destroy(&input); - if (astream->set.close_attachment_ostream(output, ret == 0, + if (astream->set.close_attachment_ostream(output, ret == 0, error_r, astream->context) < 0) ret = -1; return ret; @@ -521,7 +524,7 @@ } static int -astream_end_of_part(struct attachment_istream *astream) +astream_end_of_part(struct attachment_istream *astream, const char **error_r) { struct attachment_istream_part *part = &astream->part; size_t old_size; @@ -542,7 +545,7 @@ break; case MAIL_ATTACHMENT_STATE_YES: old_size = astream->istream.pos - astream->istream.skip; - if (astream_part_finish(astream) < 0) + if (astream_part_finish(astream, error_r) < 0) ret = -1; else { /* finished base64 may have added a few more trailing @@ -562,6 +565,7 @@ struct istream_private *stream = &astream->istream; struct message_block block; size_t old_size, new_size; + const char *error; int ret; *retry_r = FALSE; @@ -569,11 +573,16 @@ if (stream->pos - stream->skip >= stream->max_buffer_size) return -2; + if (astream->failed) { + stream->istream.stream_errno = EINVAL; + return -1; + } + old_size = stream->pos - stream->skip; switch (message_parser_parse_next_block(astream->parser, &block)) { case -1: /* done / error */ - ret = astream_end_of_part(astream); + ret = astream_end_of_part(astream, &error); if (ret > 0) { /* final data */ new_size = stream->pos - stream->skip; @@ -582,8 +591,11 @@ stream->istream.eof = TRUE; stream->istream.stream_errno = stream->parent->stream_errno; - if (ret < 0) + if (ret < 0) { + io_stream_set_error(&stream->iostream, "%s", error); stream->istream.stream_errno = EINVAL; + astream->failed = TRUE; + } astream->cur_part = NULL; return -1; case 0: @@ -595,8 +607,10 @@ if (block.part != astream->cur_part && astream->cur_part != NULL) { /* end of a MIME part */ - if (astream_end_of_part(astream) < 0) { + if (astream_end_of_part(astream, &error) < 0) { + io_stream_set_error(&stream->iostream, "%s", error); stream->istream.stream_errno = EINVAL; + astream->failed = TRUE; return -1; } } diff -r 1cf67db75455 -r 4f68ac02f46c src/lib-mail/istream-attachment-extractor.h --- a/src/lib-mail/istream-attachment-extractor.h Sun Sep 22 03:40:14 2013 +0300 +++ b/src/lib-mail/istream-attachment-extractor.h Sun Sep 22 04:14:23 2013 +0300 @@ -40,10 +40,10 @@ /* Create output stream for attachment */ int (*open_attachment_ostream)(struct istream_attachment_info *info, struct ostream **output_r, - void *context); + const char **error_r, void *context); /* Finish output stream */ int (*close_attachment_ostream)(struct ostream *output, bool success, - void *context); + const char **error_r, void *context); }; struct istream * diff -r 1cf67db75455 -r 4f68ac02f46c src/lib-mail/test-istream-attachment.c --- a/src/lib-mail/test-istream-attachment.c Sun Sep 22 03:40:14 2013 +0300 +++ b/src/lib-mail/test-istream-attachment.c Sun Sep 22 04:14:23 2013 +0300 @@ -84,6 +84,7 @@ static int test_open_attachment_ostream(struct istream_attachment_info *info, struct ostream **output_r, + const char **error_r ATTR_UNUSED, void *context ATTR_UNUSED) { struct attachment *a; @@ -106,6 +107,7 @@ } static int test_close_attachment_ostream(struct ostream *output, bool success, + const char **error_r ATTR_UNUSED, void *context ATTR_UNUSED) { struct attachment *a; diff -r 1cf67db75455 -r 4f68ac02f46c src/lib-storage/index/index-attachment.c --- a/src/lib-storage/index/index-attachment.c Sun Sep 22 03:40:14 2013 +0300 +++ b/src/lib-storage/index/index-attachment.c Sun Sep 22 04:14:23 2013 +0300 @@ -75,7 +75,8 @@ static int index_attachment_open_ostream(struct istream_attachment_info *info, - struct ostream **output_r, void *context) + struct ostream **output_r, + const char **error_r ATTR_UNUSED, void *context) { struct mail_save_context *ctx = context; struct mail_save_attachment *attach = ctx->data.attach; @@ -118,12 +119,11 @@ } static int -index_attachment_close_ostream(struct ostream *output, - bool success, void *context) +index_attachment_close_ostream(struct ostream *output, bool success, + const char **error_r, void *context) { struct mail_save_context *ctx = context; struct mail_save_attachment *attach = ctx->data.attach; - struct mail_storage *storage = ctx->transaction->box->storage; int ret = success ? 0 : -1; i_assert(attach->cur_file != NULL); @@ -131,8 +131,7 @@ if (ret < 0) fs_write_stream_abort(attach->cur_file, &output); else if (fs_write_stream_finish(attach->cur_file, &output) < 0) { - mail_storage_set_critical(storage, "%s", - fs_file_last_error(attach->cur_file)); + *error_r = t_strdup(fs_file_last_error(attach->cur_file)); ret = -1; } fs_file_deinit(&attach->cur_file); @@ -202,6 +201,9 @@ size_t size; ssize_t ret; + if (attach->input->stream_errno != 0) + return -1; + do { ret = i_stream_read(attach->input); if (ret > 0) { @@ -216,6 +218,12 @@ } } while (ret != -1); + if (attach->input->stream_errno != 0) { + mail_storage_set_critical(storage, "read(%s) failed: %s", + i_stream_get_name(attach->input), + i_stream_get_error(attach->input)); + return -1; + } if (ctx->data.output != NULL) { if (save_check_write_error(storage, ctx->data.output) < 0) return -1; @@ -229,7 +237,7 @@ (void)i_stream_read(attach->input); i_assert(attach->input->eof); - return attach->input->stream_errno == 0; + return attach->input->stream_errno == 0 ? 0 : -1; } void index_attachment_save_free(struct mail_save_context *ctx) From dovecot at dovecot.org Sun Sep 22 04:24:44 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 04:24:44 +0300 Subject: dovecot-2.2: ostream-errno: last_failed_errno wasn't set, causin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52d77a3d38b5 changeset: 16812:52d77a3d38b5 user: Timo Sirainen date: Sun Sep 22 04:24:29 2013 +0300 description: ostream-errno: last_failed_errno wasn't set, causing problems with some functions. diffstat: src/lib/ostream.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 4f68ac02f46c -r 52d77a3d38b5 src/lib/ostream.c --- a/src/lib/ostream.c Sun Sep 22 04:14:23 2013 +0300 +++ b/src/lib/ostream.c Sun Sep 22 04:24:29 2013 +0300 @@ -557,6 +557,7 @@ stream = i_new(struct ostream_private, 1); stream->ostream.closed = TRUE; stream->ostream.stream_errno = stream_errno; + stream->ostream.last_failed_errno = stream_errno; output = o_stream_create(stream, NULL, -1); o_stream_set_name(output, "(error)"); From dovecot at dovecot.org Sun Sep 22 04:42:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 04:42:07 +0300 Subject: dovecot-2.2: lib-storage: Improve error message logging when att... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/38382fa18285 changeset: 16814:38382fa18285 user: Timo Sirainen date: Sun Sep 22 04:40:02 2013 +0300 description: lib-storage: Improve error message logging when attachment writing fails diffstat: src/lib-storage/index/index-attachment.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r a919c43ab91b -r 38382fa18285 src/lib-storage/index/index-attachment.c --- a/src/lib-storage/index/index-attachment.c Sun Sep 22 04:39:35 2013 +0300 +++ b/src/lib-storage/index/index-attachment.c Sun Sep 22 04:40:02 2013 +0300 @@ -131,7 +131,9 @@ if (ret < 0) fs_write_stream_abort(attach->cur_file, &output); else if (fs_write_stream_finish(attach->cur_file, &output) < 0) { - *error_r = t_strdup(fs_file_last_error(attach->cur_file)); + *error_r = t_strdup_printf("Couldn't create attachment %s: %s", + fs_file_path(attach->cur_file), + fs_file_last_error(attach->cur_file)); ret = -1; } fs_file_deinit(&attach->cur_file); From dovecot at dovecot.org Sun Sep 22 04:42:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 04:42:06 +0300 Subject: dovecot-2.2: lib: Added [io]_stream_create_error_str() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a919c43ab91b changeset: 16813:a919c43ab91b user: Timo Sirainen date: Sun Sep 22 04:39:35 2013 +0300 description: lib: Added [io]_stream_create_error_str() diffstat: src/lib/iostream-private.h | 2 ++ src/lib/iostream.c | 8 +++++++- src/lib/istream.c | 13 +++++++++++++ src/lib/istream.h | 3 +++ src/lib/ostream.c | 14 ++++++++++++++ src/lib/ostream.h | 3 +++ 6 files changed, 42 insertions(+), 1 deletions(-) diffs (99 lines): diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/iostream-private.h --- a/src/lib/iostream-private.h Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/iostream-private.h Sun Sep 22 04:39:35 2013 +0300 @@ -33,5 +33,7 @@ Setting this error replaces the previously set error. */ void io_stream_set_error(struct iostream_private *stream, const char *fmt, ...) ATTR_FORMAT(2, 3); +void io_stream_set_verror(struct iostream_private *stream, + const char *fmt, va_list args) ATTR_FORMAT(2, 0); #endif diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/iostream.c --- a/src/lib/iostream.c Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/iostream.c Sun Sep 22 04:39:35 2013 +0300 @@ -68,7 +68,13 @@ va_list args; va_start(args, fmt); + io_stream_set_verror(stream, fmt, args); + va_end(args); +} + +void io_stream_set_verror(struct iostream_private *stream, + const char *fmt, va_list args) +{ i_free(stream->error); stream->error = i_strdup_vprintf(fmt, args); - va_end(args); } diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/istream.c --- a/src/lib/istream.c Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/istream.c Sun Sep 22 04:39:35 2013 +0300 @@ -766,3 +766,16 @@ i_stream_set_name(&stream->istream, "(error)"); return &stream->istream; } + +struct istream * +i_stream_create_error_str(int stream_errno, const char *fmt, ...) +{ + struct istream *input; + va_list args; + + va_start(args, fmt); + input = i_stream_create_error(stream_errno); + io_stream_set_verror(&input->real_stream->iostream, fmt, args); + va_end(args); + return input; +} diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/istream.h --- a/src/lib/istream.h Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/istream.h Sun Sep 22 04:39:35 2013 +0300 @@ -35,6 +35,9 @@ struct istream *i_stream_create_range(struct istream *input, uoff_t v_offset, uoff_t v_size); struct istream *i_stream_create_error(int stream_errno); +struct istream * +i_stream_create_error_str(int stream_errno, const char *fmt, ...) + ATTR_FORMAT(2, 3); /* Set name (e.g. path) for input stream. */ void i_stream_set_name(struct istream *stream, const char *name); diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/ostream.c --- a/src/lib/ostream.c Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/ostream.c Sun Sep 22 04:39:35 2013 +0300 @@ -560,6 +560,20 @@ stream->ostream.last_failed_errno = stream_errno; output = o_stream_create(stream, NULL, -1); + o_stream_set_no_error_handling(output, TRUE); o_stream_set_name(output, "(error)"); return output; } + +struct ostream * +o_stream_create_error_str(int stream_errno, const char *fmt, ...) +{ + struct ostream *output; + va_list args; + + va_start(args, fmt); + output = o_stream_create_error(stream_errno); + io_stream_set_verror(&output->real_stream->iostream, fmt, args); + va_end(args); + return output; +} diff -r 52d77a3d38b5 -r a919c43ab91b src/lib/ostream.h --- a/src/lib/ostream.h Sun Sep 22 04:24:29 2013 +0300 +++ b/src/lib/ostream.h Sun Sep 22 04:39:35 2013 +0300 @@ -38,6 +38,9 @@ struct ostream *o_stream_create_buffer(buffer_t *buf); /* Create an output streams that always fails the writes. */ struct ostream *o_stream_create_error(int stream_errno); +struct ostream * +o_stream_create_error_str(int stream_errno, const char *fmt, ...) + ATTR_FORMAT(2, 3); /* Set name (e.g. path) for output stream. */ void o_stream_set_name(struct ostream *stream, const char *name); From dovecot at dovecot.org Sun Sep 22 04:42:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 04:42:07 +0300 Subject: dovecot-2.2: lib-fs: Improved fs_write_*() error messages. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1cd94c4cd456 changeset: 16815:1cd94c4cd456 user: Timo Sirainen date: Sun Sep 22 04:41:50 2013 +0300 description: lib-fs: Improved fs_write_*() error messages. diffstat: src/lib-fs/fs-posix.c | 14 ++++++++++---- src/lib-fs/fs-sis-queue.c | 8 +++++--- src/lib-fs/fs-sis.c | 7 ++++--- 3 files changed, 19 insertions(+), 10 deletions(-) diffs (74 lines): diff -r 38382fa18285 -r 1cd94c4cd456 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Sun Sep 22 04:40:02 2013 +0300 +++ b/src/lib-fs/fs-posix.c Sun Sep 22 04:41:50 2013 +0300 @@ -47,6 +47,7 @@ bool seek_to_beginning; bool success; + bool open_failed; }; struct posix_fs_lock { @@ -462,7 +463,9 @@ file->write_buf = buffer_create_dynamic(default_pool, 1024*32); _file->output = o_stream_create_buffer(file->write_buf); } else if (file->fd == -1 && fs_posix_open(file) < 0) { - _file->output = o_stream_create_error(errno); + _file->output = o_stream_create_error_str(errno, "%s", + fs_file_last_error(_file)); + file->open_failed = TRUE; } else { _file->output = o_stream_create_fd_file(file->fd, (uoff_t)-1, FALSE); @@ -475,9 +478,12 @@ struct posix_fs_file *file = (struct posix_fs_file *)_file; int ret = success ? 0 : -1; - if (o_stream_nfinish(_file->output) < 0) { - fs_set_error(_file->fs, "write(%s) failed: %m", - o_stream_get_name(_file->output)); + if (file->open_failed) + ret = -1; + else if (o_stream_nfinish(_file->output) < 0) { + fs_set_error(_file->fs, "write(%s) failed: %s", + o_stream_get_name(_file->output), + o_stream_get_error(_file->output)); ret = -1; } o_stream_destroy(&_file->output); diff -r 38382fa18285 -r 1cd94c4cd456 src/lib-fs/fs-sis-queue.c --- a/src/lib-fs/fs-sis-queue.c Sun Sep 22 04:40:02 2013 +0300 +++ b/src/lib-fs/fs-sis-queue.c Sun Sep 22 04:41:50 2013 +0300 @@ -234,10 +234,12 @@ i_assert(_file->output == NULL); - if (file->super == NULL) - _file->output = o_stream_create_error(EINVAL); - else + if (file->super == NULL) { + _file->output = o_stream_create_error_str(EINVAL, "%s", + fs_file_last_error(_file)); + } else { _file->output = fs_write_stream(file->super); + } o_stream_set_name(_file->output, _file->path); } diff -r 38382fa18285 -r 1cd94c4cd456 src/lib-fs/fs-sis.c --- a/src/lib-fs/fs-sis.c Sun Sep 22 04:40:02 2013 +0300 +++ b/src/lib-fs/fs-sis.c Sun Sep 22 04:41:50 2013 +0300 @@ -354,9 +354,10 @@ i_assert(_file->output == NULL); - if (file->super == NULL) - _file->output = o_stream_create_error(EINVAL); - else { + if (file->super == NULL) { + _file->output = o_stream_create_error_str(EINVAL, "%s", + fs_file_last_error(_file)); + } else { file->fs_output = fs_write_stream(file->super); if (file->hash_input == NULL) _file->output = file->fs_output; From dovecot at dovecot.org Sun Sep 22 06:31:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 06:31:01 +0300 Subject: dovecot-2.2: mbox: Fixed assert-crash due to wrong transaction_c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/afc8c800f6ca changeset: 16816:afc8c800f6ca user: Timo Sirainen date: Sun Sep 22 06:30:47 2013 +0300 description: mbox: Fixed assert-crash due to wrong transaction_count handling. This happened only when messages had been expunged by another session just before saving a new message. diffstat: src/lib-storage/index/mbox/mbox-storage.c | 3 ++- src/lib-storage/mail-storage.c | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diffs (61 lines): diff -r 1cd94c4cd456 -r afc8c800f6ca src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 22 04:41:50 2013 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 22 06:30:47 2013 +0300 @@ -737,7 +737,8 @@ if (lock_id2 != 0) mbox_unlock(mbox, lock_id2); if (mbox->mbox_global_lock_id == 0) { - i_assert(mbox->box.transaction_count > 0 || + i_assert(mbox->box.transaction_count > 0); + i_assert(mbox->box.transaction_count > 1 || mbox->external_transactions > 0 || mbox->mbox_lock_type == F_UNLCK); } else { diff -r 1cd94c4cd456 -r afc8c800f6ca src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Sun Sep 22 04:41:50 2013 +0300 +++ b/src/lib-storage/mail-storage.c Sun Sep 22 06:30:47 2013 +0300 @@ -1836,15 +1836,15 @@ struct mail_transaction_commit_changes *changes_r) { struct mailbox_transaction_context *t = *_t; + struct mailbox *box = t->box; unsigned int save_count = t->save_count; int ret; - t->box->transaction_count--; changes_r->pool = NULL; *_t = NULL; T_BEGIN { - ret = t->box->v.transaction_commit(t, changes_r); + ret = box->v.transaction_commit(t, changes_r); } T_END; /* either all the saved messages get UIDs or none, because a) we failed, b) MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS not set, @@ -1852,6 +1852,11 @@ i_assert(ret < 0 || seq_range_count(&changes_r->saved_uids) == save_count || array_count(&changes_r->saved_uids) == 0); + /* decrease the transaction count only after transaction_commit(). + that way if it creates and destroys transactions internally, we + don't see transaction_count=0 until the parent transaction is fully + finished */ + box->transaction_count--; if (ret < 0 && changes_r->pool != NULL) pool_unref(&changes_r->pool); return ret; @@ -1860,11 +1865,11 @@ void mailbox_transaction_rollback(struct mailbox_transaction_context **_t) { struct mailbox_transaction_context *t = *_t; - - t->box->transaction_count--; + struct mailbox *box = t->box; *_t = NULL; - t->box->v.transaction_rollback(t); + box->v.transaction_rollback(t); + box->transaction_count--; } unsigned int mailbox_transaction_get_count(const struct mailbox *box) From dovecot at dovecot.org Sun Sep 22 07:20:24 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 07:20:24 +0300 Subject: dovecot-2.2: istream-sized: Fail read() also when the stream is ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5b1aac85eb17 changeset: 16817:5b1aac85eb17 user: Timo Sirainen date: Sun Sep 22 07:19:39 2013 +0300 description: istream-sized: Fail read() also when the stream is larger than it should have been. diffstat: src/lib/istream-sized.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (34 lines): diff -r afc8c800f6ca -r 5b1aac85eb17 src/lib/istream-sized.c --- a/src/lib/istream-sized.c Sun Sep 22 06:30:47 2013 +0300 +++ b/src/lib/istream-sized.c Sun Sep 22 07:19:39 2013 +0300 @@ -8,6 +8,7 @@ struct istream_private istream; uoff_t size; + bool failed; }; static ssize_t i_stream_sized_read(struct istream_private *stream) @@ -18,6 +19,12 @@ ssize_t ret; size_t pos; + if (sstream->failed) { + /* avoid duplicate errors */ + stream->istream.stream_errno = EINVAL; + return -1; + } + if (stream->istream.v_offset + (stream->pos - stream->skip) >= sstream->size) { stream->istream.eof = TRUE; @@ -56,6 +63,9 @@ stream->iostream.error); pos = left; stream->istream.eof = TRUE; + stream->istream.stream_errno = EINVAL; + sstream->failed = TRUE; + return -1; } else if (!stream->istream.eof) { /* still more to read */ } else if (stream->istream.stream_errno == ENOENT) { From dovecot at dovecot.org Sun Sep 22 07:20:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 07:20:25 +0300 Subject: dovecot-2.2: istream-attachment-connector: Add base64 parameters... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be1d8dd8be94 changeset: 16818:be1d8dd8be94 user: Timo Sirainen date: Sun Sep 22 07:20:12 2013 +0300 description: istream-attachment-connector: Add base64 parameters to stream name for error messages. diffstat: src/lib-mail/istream-attachment-connector.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 5b1aac85eb17 -r be1d8dd8be94 src/lib-mail/istream-attachment-connector.c --- a/src/lib-mail/istream-attachment-connector.c Sun Sep 22 07:19:39 2013 +0300 +++ b/src/lib-mail/istream-attachment-connector.c Sun Sep 22 07:20:12 2013 +0300 @@ -81,6 +81,10 @@ input = i_stream_create_base64_encoder(decoded_input, base64_blocks_per_line*4, base64_have_crlf); + i_stream_set_name(input, t_strdup_printf("%s[base64:%u b/l%s]", + i_stream_get_name(decoded_input), + base64_blocks_per_line, + base64_have_crlf ? ",crlf" : "")); } input2 = i_stream_create_sized(input, encoded_size); array_append(&conn->streams, &input2, 1); From dovecot at dovecot.org Sun Sep 22 07:24:38 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 07:24:38 +0300 Subject: dovecot-2.2: i_getpw*(): Added OpenBSD workaround. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/801714cba91b changeset: 16819:801714cba91b user: Timo Sirainen date: Sun Sep 22 07:24:26 2013 +0300 description: i_getpw*(): Added OpenBSD workaround. diffstat: src/lib/ipwd.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (27 lines): diff -r be1d8dd8be94 -r 801714cba91b src/lib/ipwd.c --- a/src/lib/ipwd.c Sun Sep 22 07:20:12 2013 +0300 +++ b/src/lib/ipwd.c Sun Sep 22 07:24:26 2013 +0300 @@ -50,6 +50,11 @@ do { pw_init(); errno = getpwnam_r(name, pwd_r, pwbuf, pwbuf_size, &result); +#ifdef __OpenBSD__ + /* OpenBSD returns 1 for all errors, assume it's ERANGE */ + if (errno == 1) + errno = ERANGE; +#endif } while (errno == ERANGE); if (result != NULL) return 1; @@ -68,6 +73,11 @@ do { pw_init(); errno = getpwuid_r(uid, pwd_r, pwbuf, pwbuf_size, &result); +#ifdef __OpenBSD__ + /* OpenBSD returns 1 for all errors, assume it's ERANGE */ + if (errno == 1) + errno = ERANGE; +#endif } while (errno == ERANGE); if (result != NULL) return 1; From dovecot at dovecot.org Sun Sep 22 07:40:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 07:40:15 +0300 Subject: dovecot-2.2: maildir: Autocreate missing cur/ new/ tmp/ director... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6942a3da740c changeset: 16820:6942a3da740c user: Timo Sirainen date: Sun Sep 22 07:40:01 2013 +0300 description: maildir: Autocreate missing cur/ new/ tmp/ directories only with Maildir++ and imapdir. \Noselect mailboxes aren't possible with those layouts, but with other layouts they are and they shouldn't get automatically created just by selecting them. diffstat: src/lib-storage/index/maildir/maildir-storage.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 801714cba91b -r 6942a3da740c src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Sun Sep 22 07:24:26 2013 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sun Sep 22 07:40:01 2013 +0300 @@ -351,7 +351,14 @@ if (ret < 0) return -1; - /* tmp/ directory doesn't exist. does the maildir? */ + /* tmp/ directory doesn't exist. does the maildir? autocreate missing + dirs only with Maildir++ and imapdir layouts. */ + if (strcmp(box->list->name, MAILBOX_LIST_NAME_MAILDIRPLUSPLUS) != 0 && + strcmp(box->list->name, MAILBOX_LIST_NAME_IMAPDIR) != 0) { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(box->vname)); + return -1; + } root_dir = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX); if (strcmp(box_path, root_dir) == 0) { From dovecot at dovecot.org Sun Sep 22 07:43:37 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 22 Sep 2013 07:43:37 +0300 Subject: dovecot-2.2: imapc: Don't crash if imapc_password is missing. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a5d92b8d3d5 changeset: 16821:1a5d92b8d3d5 user: Timo Sirainen date: Sun Sep 22 07:43:31 2013 +0300 description: imapc: Don't crash if imapc_password is missing. diffstat: src/lib-storage/index/imapc/imapc-list.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (17 lines): diff -r 6942a3da740c -r 1a5d92b8d3d5 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Sun Sep 22 07:40:01 2013 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Sun Sep 22 07:43:31 2013 +0300 @@ -100,9 +100,10 @@ /* make sure all pending commands are aborted before anything is deinitialized */ - imapc_client_disconnect(list->client->client); - - imapc_storage_client_unref(&list->client); + if (list->client != NULL) { + imapc_client_disconnect(list->client->client); + imapc_storage_client_unref(&list->client); + } if (list->index_list != NULL) mailbox_list_destroy(&list->index_list); mailbox_tree_deinit(&list->mailboxes); From pigeonhole at rename-it.nl Sun Sep 22 12:51:46 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 22 Sep 2013 11:51:46 +0200 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Fixed the `Invalid... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/3163f3696498 changeset: 1805:3163f3696498 user: Stephan Bosch date: Sun Sep 22 11:51:38 2013 +0200 description: doveadm sieve plugin: Fixed the `Invalid value for default sieve attribute' problem. The attribute type code data was allocated on the stack rather than statically, causing the returned stream to point to invalid data. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 34a5db66fc52 -r 3163f3696498 src/plugins/doveadm-sieve/doveadm-sieve-plugin.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Tue Sep 17 23:57:05 2013 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-plugin.c Sun Sep 22 11:51:38 2013 +0200 @@ -388,7 +388,7 @@ bool add_type_prefix, struct mail_attribute_value *value_r, const char **errorstr_r) { - char type = MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_SCRIPT; + static char type = MAILBOX_ATTRIBUTE_SIEVE_DEFAULT_SCRIPT; struct istream *input, *inputs[3]; const struct stat *st; enum sieve_error error; From dovecot at dovecot.org Mon Sep 23 09:45:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 23 Sep 2013 09:45:23 +0300 Subject: dovecot-2.2: lib: Added net_listen_full() with a flag to set SO_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6a814345f16c changeset: 16822:6a814345f16c user: Timo Sirainen date: Mon Sep 23 04:06:08 2013 +0300 description: lib: Added net_listen_full() with a flag to set SO_REUSEPORT on the socket if available. diffstat: src/lib/net.c | 16 ++++++++++++++++ src/lib/net.h | 8 ++++++++ 2 files changed, 24 insertions(+), 0 deletions(-) diffs (58 lines): diff -r 1a5d92b8d3d5 -r 6a814345f16c src/lib/net.c --- a/src/lib/net.c Sun Sep 22 07:43:31 2013 +0300 +++ b/src/lib/net.c Mon Sep 23 04:06:08 2013 +0300 @@ -384,6 +384,14 @@ int net_listen(const struct ip_addr *my_ip, unsigned int *port, int backlog) { + enum net_listen_flags flags = 0; + + return net_listen_full(my_ip, port, &flags, backlog); +} + +int net_listen_full(const struct ip_addr *my_ip, unsigned int *port, + enum net_listen_flags *flags, int backlog) +{ union sockaddr_union so; int ret, fd, opt = 1; socklen_t len; @@ -413,6 +421,14 @@ setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &opt, sizeof(opt)); + if ((*flags & NET_LISTEN_FLAG_REUSEPORT) != 0) { +#ifdef SO_REUSEPORT + if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, + &opt, sizeof(opt)) < 0) +#endif + *flags &= ~NET_LISTEN_FLAG_REUSEPORT; + } + /* If using IPv6, bind only to IPv6 if possible. This avoids ambiguities with IPv4-mapped IPv6 addresses. */ #ifdef IPV6_V6ONLY diff -r 1a5d92b8d3d5 -r 6a814345f16c src/lib/net.h --- a/src/lib/net.h Sun Sep 22 07:43:31 2013 +0300 +++ b/src/lib/net.h Mon Sep 23 04:06:08 2013 +0300 @@ -55,6 +55,12 @@ #define IPADDR_IS_V6(ip) ((ip)->family == AF_INET6) #define IPADDR_BITS(ip) (IPADDR_IS_V4(ip) ? 32 : 128) +enum net_listen_flags { + /* Try to use SO_REUSEPORT if available. If it's not, this flag is + cleared on return. */ + NET_LISTEN_FLAG_REUSEPORT = 0x01 +}; + /* Returns TRUE if IPs are the same */ bool net_ip_compare(const struct ip_addr *ip1, const struct ip_addr *ip2); /* Returns 0 if IPs are the same, -1 or 1 otherwise. */ @@ -91,6 +97,8 @@ /* Listen for connections on a socket */ int net_listen(const struct ip_addr *my_ip, unsigned int *port, int backlog); +int net_listen_full(const struct ip_addr *my_ip, unsigned int *port, + enum net_listen_flags *flags, int backlog); /* Listen for connections on an UNIX socket */ int net_listen_unix(const char *path, int backlog); /* Like net_listen_unix(), but if socket already exists, try to connect to it. From dovecot at dovecot.org Mon Sep 23 09:45:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 23 Sep 2013 09:45:23 +0300 Subject: dovecot-2.2: master: Added reuse_port setting to inet_listeners,... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a991a0547daa changeset: 16823:a991a0547daa user: Timo Sirainen date: Mon Sep 23 04:25:16 2013 +0300 description: master: Added reuse_port setting to inet_listeners, which enables SO_REUSEPORT if available. After forking a new service process, a new listener socket is created for each such inet_listener. Linux v3.9+ added SO_REUSEPORT feature, which should distribute clients more uniformly to the processes. I'm not sure if this makes any difference in BSDs. At least in Linux v3.9 there was still a bug that if the number of listening processes changed, some TCP handshakes might not finish. I don't see if this has already been fixed, so this is probably safe to use only for services whose process count doesn't change (e.g. process_min_avail is set high enough). diffstat: src/imap-login/imap-login-settings.c | 4 ++-- src/lib-master/service-settings.h | 1 + src/master/master-settings.c | 4 +++- src/master/service-listen.c | 35 +++++++++++++++++++++-------------- src/master/service-listen.h | 2 ++ src/master/service-process.c | 20 ++++++++++++++++++++ src/master/service.h | 2 ++ src/pop3-login/pop3-login-settings.c | 4 ++-- 8 files changed, 53 insertions(+), 19 deletions(-) diffs (200 lines): diff -r 6a814345f16c -r a991a0547daa src/imap-login/imap-login-settings.c --- a/src/imap-login/imap-login-settings.c Mon Sep 23 04:06:08 2013 +0300 +++ b/src/imap-login/imap-login-settings.c Mon Sep 23 04:25:16 2013 +0300 @@ -11,8 +11,8 @@ /* */ static struct inet_listener_settings imap_login_inet_listeners_array[] = { - { "imap", "", 143, FALSE }, - { "imaps", "", 993, TRUE } + { .name = "imap", .address = "", .port = 143 }, + { .name = "imaps", .address = "", .port = 993, .ssl = TRUE } }; static struct inet_listener_settings *imap_login_inet_listeners[] = { &imap_login_inet_listeners_array[0], diff -r 6a814345f16c -r a991a0547daa src/lib-master/service-settings.h --- a/src/lib-master/service-settings.h Mon Sep 23 04:06:08 2013 +0300 +++ b/src/lib-master/service-settings.h Mon Sep 23 04:25:16 2013 +0300 @@ -31,6 +31,7 @@ const char *address; unsigned int port; bool ssl; + bool reuse_port; }; ARRAY_DEFINE_TYPE(inet_listener_settings, struct inet_listener_settings *); diff -r 6a814345f16c -r a991a0547daa src/master/master-settings.c --- a/src/master/master-settings.c Mon Sep 23 04:06:08 2013 +0300 +++ b/src/master/master-settings.c Mon Sep 23 04:25:16 2013 +0300 @@ -64,6 +64,7 @@ DEF(SET_STR, address), DEF(SET_UINT, port), DEF(SET_BOOL, ssl), + DEF(SET_BOOL, reuse_port), SETTING_DEFINE_LIST_END }; @@ -72,7 +73,8 @@ .name = "", .address = "", .port = 0, - .ssl = FALSE + .ssl = FALSE, + .reuse_port = FALSE }; static const struct setting_parser_info inet_listener_setting_parser_info = { diff -r 6a814345f16c -r a991a0547daa src/master/service-listen.c --- a/src/master/service-listen.c Mon Sep 23 04:06:08 2013 +0300 +++ b/src/master/service-listen.c Mon Sep 23 04:25:16 2013 +0300 @@ -189,9 +189,11 @@ static int service_inet_listener_listen(struct service_listener *l) { struct service *service = l->service; + enum net_listen_flags flags = 0; const struct inet_listener_settings *set = l->set.inetset.set; unsigned int port = set->port; int fd; + #ifdef HAVE_SYSTEMD if (systemd_listen_fd(&l->set.inetset.ip, port, &fd) < 0) return -1; @@ -199,13 +201,16 @@ if (fd == -1) #endif { - fd = net_listen(&l->set.inetset.ip, &port, - service_get_backlog(service)); + if (set->reuse_port) + flags |= NET_LISTEN_FLAG_REUSEPORT; + fd = net_listen_full(&l->set.inetset.ip, &port, &flags, + service_get_backlog(service)); if (fd < 0) { service_error(service, "listen(%s, %u) failed: %m", l->inet_address, set->port); return errno == EADDRINUSE ? 0 : -1; } + l->reuse_port = (flags & NET_LISTEN_FLAG_REUSEPORT) != 0; } net_set_nonblock(fd, TRUE); fd_close_on_exec(fd, TRUE); @@ -214,6 +219,19 @@ return 1; } +int service_listener_listen(struct service_listener *l) +{ + switch (l->type) { + case SERVICE_LISTENER_UNIX: + return service_unix_listener_listen(l); + case SERVICE_LISTENER_FIFO: + return service_fifo_listener_listen(l); + case SERVICE_LISTENER_INET: + return service_inet_listener_listen(l); + } + i_unreached(); +} + static int service_listen(struct service *service) { struct service_listener *const *listeners; @@ -225,18 +243,7 @@ if (l->fd != -1) continue; - switch (l->type) { - case SERVICE_LISTENER_UNIX: - ret2 = service_unix_listener_listen(l); - break; - case SERVICE_LISTENER_FIFO: - ret2 = service_fifo_listener_listen(l); - break; - case SERVICE_LISTENER_INET: - ret2 = service_inet_listener_listen(l); - break; - } - + ret2 = service_listener_listen(l); if (ret2 < ret) ret = ret2; } diff -r 6a814345f16c -r a991a0547daa src/master/service-listen.h --- a/src/master/service-listen.h Mon Sep 23 04:06:08 2013 +0300 +++ b/src/master/service-listen.h Mon Sep 23 04:25:16 2013 +0300 @@ -13,4 +13,6 @@ int services_listen_using(struct service_list *new_service_list, struct service_list *old_service_list); +int service_listener_listen(struct service_listener *l); + #endif diff -r 6a814345f16c -r a991a0547daa src/master/service-process.c --- a/src/master/service-process.c Mon Sep 23 04:06:08 2013 +0300 +++ b/src/master/service-process.c Mon Sep 23 04:25:16 2013 +0300 @@ -23,6 +23,7 @@ #include "dup2-array.h" #include "service.h" #include "service-anvil.h" +#include "service-listen.h" #include "service-log.h" #include "service-process-notify.h" #include "service-process.h" @@ -34,6 +35,24 @@ #include #include +static void service_reopen_inet_listeners(struct service *service) +{ + struct service_listener *const *listeners; + unsigned int i, count; + int old_fd; + + listeners = array_get(&service->listeners, &count); + for (i = 0; i < count; i++) { + if (!listeners[i]->reuse_port || listeners[i]->fd == -1) + continue; + + old_fd = listeners[i]->fd; + listeners[i]->fd = -1; + if (service_listener_listen(listeners[i]) < 0) + listeners[i]->fd = old_fd; + } +} + static void service_dup_fds(struct service *service) { @@ -305,6 +324,7 @@ if (pid == 0) { /* child */ service_process_setup_environment(service, uid); + service_reopen_inet_listeners(service); service_dup_fds(service); drop_privileges(service); process_exec(service->executable, NULL); diff -r 6a814345f16c -r a991a0547daa src/master/service.h --- a/src/master/service.h Mon Sep 23 04:06:08 2013 +0300 +++ b/src/master/service.h Mon Sep 23 04:25:16 2013 +0300 @@ -38,6 +38,8 @@ struct ip_addr ip; } inetset; } set; + + bool reuse_port; }; struct service { diff -r 6a814345f16c -r a991a0547daa src/pop3-login/pop3-login-settings.c --- a/src/pop3-login/pop3-login-settings.c Mon Sep 23 04:06:08 2013 +0300 +++ b/src/pop3-login/pop3-login-settings.c Mon Sep 23 04:25:16 2013 +0300 @@ -11,8 +11,8 @@ /* */ static struct inet_listener_settings pop3_login_inet_listeners_array[] = { - { "pop3", "", 110, FALSE }, - { "pop3s", "", 995, TRUE } + { .name = "pop3", .address = "", .port = 110 }, + { .name = "pop3s", .address = "", .port = 995, .ssl = TRUE } }; static struct inet_listener_settings *pop3_login_inet_listeners[] = { &pop3_login_inet_listeners_array[0], From pigeonhole at rename-it.nl Mon Sep 23 22:00:11 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 23 Sep 2013 21:00:11 +0200 Subject: dovecot-2.2-pigeonhole: managesieve: Updated to changes in Dovec... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9b78ef1584c6 changeset: 1806:9b78ef1584c6 user: Stephan Bosch date: Mon Sep 23 20:58:02 2013 +0200 description: managesieve: Updated to changes in Dovecot listeners regarding SO_REUSEPORT. diffstat: src/managesieve-login/managesieve-login-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3163f3696498 -r 9b78ef1584c6 src/managesieve-login/managesieve-login-settings.c --- a/src/managesieve-login/managesieve-login-settings.c Sun Sep 22 11:51:38 2013 +0200 +++ b/src/managesieve-login/managesieve-login-settings.c Mon Sep 23 20:58:02 2013 +0200 @@ -23,7 +23,7 @@ /* */ static struct inet_listener_settings managesieve_login_inet_listeners_array[] = { - { "sieve", "", 4190, FALSE }, + { "sieve", "", 4190, FALSE, FALSE }, }; static struct inet_listener_settings *managesieve_login_inet_listeners[] = { &managesieve_login_inet_listeners_array[0] From pigeonhole at rename-it.nl Mon Sep 23 23:03:05 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 23 Sep 2013 22:03:05 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: vacation extension: Added sup... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/4924724b7f4f changeset: 1807:4924724b7f4f user: Stephan Bosch date: Mon Sep 23 22:01:38 2013 +0200 description: lib-sieve: vacation extension: Added support for sending vacation messages with an actual sender. Use sieve_vacation_send_from_recipient to enable this behavior. Documentation in doc/extensions/vacation.txt is updated. diffstat: doc/extensions/vacation.txt | 8 + src/lib-sieve/plugins/vacation/cmd-vacation.c | 20 ++- src/lib-sieve/plugins/vacation/ext-vacation-common.c | 8 +- src/lib-sieve/plugins/vacation/ext-vacation-common.h | 1 + tests/extensions/vacation/message.svtest | 90 ++++++++++++++++++++ 5 files changed, 119 insertions(+), 8 deletions(-) diffs (259 lines): diff -r 9b78ef1584c6 -r 4924724b7f4f doc/extensions/vacation.txt --- a/doc/extensions/vacation.txt Mon Sep 23 20:58:02 2013 +0200 +++ b/doc/extensions/vacation.txt Mon Sep 23 22:01:38 2013 +0200 @@ -73,6 +73,14 @@ Sieve standards and can cause vacation replies to be sent for messages not directly addressed at the recipient. +sieve_vacation_send_from_recipient = no + This setting determines whether vacation messages are sent with the SMTP MAIL + FROM envelope address set to the recipient address of the Sieve script owner. + Normally this is set to <>, which is the default as recommended in the + specification. This is meant to prevent mail loops. However, there are + situations for which a valid sender address is required and this setting can + be used to accommodate for those. + Invalid values for the settings above will make the Sieve interpreter log a warning and revert to the default values. diff -r 9b78ef1584c6 -r 4924724b7f4f src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Sep 23 20:58:02 2013 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Mon Sep 23 22:01:38 2013 +0200 @@ -897,16 +897,15 @@ static bool act_vacation_send (const struct sieve_action_exec_env *aenv, struct act_vacation_context *ctx, - const char *reply_to, const char *reply_from) + const char *reply_to, const char *reply_from, const char *smtp_from) { const struct sieve_message_data *msgdata = aenv->msgdata; const struct sieve_script_env *senv = aenv->scriptenv; void *smtp_handle; struct ostream *output; string_t *msg; - const char *outmsgid; const char *const *headers; - const char *subject; + const char *outmsgid, *subject; int ret; /* Check smpt functions just to be sure */ @@ -934,7 +933,7 @@ /* Open smtp session */ - smtp_handle = sieve_smtp_open(senv, reply_to, NULL, &output); + smtp_handle = sieve_smtp_open(senv, reply_to, smtp_from, &output); outmsgid = sieve_message_get_new_id(aenv->svinst); /* Produce a proper reply */ @@ -1035,7 +1034,7 @@ const char *recipient = sieve_message_get_final_recipient(aenv->msgctx); const char *const *hdsp; const char *const *headers; - const char *reply_from = NULL, *orig_recipient = NULL; + const char *reply_from = NULL, *orig_recipient = NULL, *smtp_from = NULL; bool result; /* Is the recipient unset? @@ -1162,6 +1161,7 @@ /* Final recipient directly listed in headers? */ if ( _contains_my_address(headers, recipient) ) { reply_from = recipient; + smtp_from = recipient; break; } @@ -1169,6 +1169,7 @@ if ( orig_recipient != NULL && _contains_my_address(headers, orig_recipient) ) { reply_from = orig_recipient; + smtp_from = orig_recipient; break; } @@ -1178,8 +1179,12 @@ const char * const *my_address = ctx->addresses; while ( !found && *my_address != NULL ) { - if ( (found=_contains_my_address(headers, *my_address)) ) + if ( (found=_contains_my_address(headers, *my_address)) ) { reply_from = *my_address; + /* Avoid letting user determine SMTP sender directly */ + smtp_from = + ( orig_recipient == NULL ? recipient : orig_recipient ); + } my_address++; } @@ -1220,7 +1225,8 @@ /* Send the message */ T_BEGIN { - result = act_vacation_send(aenv, ctx, sender, reply_from); + result = act_vacation_send(aenv, ctx, sender, reply_from, + (config->send_from_recipient ? smtp_from : NULL)); } T_END; if ( result ) { diff -r 9b78ef1584c6 -r 4924724b7f4f src/lib-sieve/plugins/vacation/ext-vacation-common.c --- a/src/lib-sieve/plugins/vacation/ext-vacation-common.c Mon Sep 23 20:58:02 2013 +0200 +++ b/src/lib-sieve/plugins/vacation/ext-vacation-common.c Mon Sep 23 22:01:38 2013 +0200 @@ -16,7 +16,7 @@ struct sieve_instance *svinst = ext->svinst; struct ext_vacation_config *config; sieve_number_t min_period, max_period, default_period; - bool use_original_recipient, dont_check_recipient; + bool use_original_recipient, dont_check_recipient, send_from_recipient; if ( *context != NULL ) { ext_vacation_unload(ext); @@ -60,12 +60,18 @@ dont_check_recipient = FALSE; } + if ( !sieve_setting_get_bool_value + (svinst, "sieve_vacation_send_from_recipient", &send_from_recipient) ) { + send_from_recipient = FALSE; + } + config = i_new(struct ext_vacation_config, 1); config->min_period = min_period; config->max_period = max_period; config->default_period = default_period; config->use_original_recipient = use_original_recipient; config->dont_check_recipient = dont_check_recipient; + config->send_from_recipient = send_from_recipient; *context = (void *) config; diff -r 9b78ef1584c6 -r 4924724b7f4f src/lib-sieve/plugins/vacation/ext-vacation-common.h --- a/src/lib-sieve/plugins/vacation/ext-vacation-common.h Mon Sep 23 20:58:02 2013 +0200 +++ b/src/lib-sieve/plugins/vacation/ext-vacation-common.h Mon Sep 23 22:01:38 2013 +0200 @@ -20,6 +20,7 @@ unsigned int default_period; bool use_original_recipient; bool dont_check_recipient; + bool send_from_recipient; }; /* diff -r 9b78ef1584c6 -r 4924724b7f4f tests/extensions/vacation/message.svtest --- a/tests/extensions/vacation/message.svtest Mon Sep 23 20:58:02 2013 +0200 +++ b/tests/extensions/vacation/message.svtest Mon Sep 23 22:01:38 2013 +0200 @@ -1,8 +1,13 @@ require "vnd.dovecot.testsuite"; require "vacation"; require "variables"; +require "envelope"; require "body"; +/* + * References + */ + test_set "message" text: From: stephan at example.org Subject: frop @@ -43,12 +48,20 @@ } } +/* + * In-Reply-To + */ + test "In-Reply-To" { if not header :is "in-reply-to" "<432df324 at example.org>" { test_fail "in-reply-to header set incorrectly"; } } +/* + * Variables + */ + test_result_reset; test_set "message" text: @@ -88,3 +101,80 @@ test_fail "message not set properly"; } } + +/* + * NULL Sender + */ + +test_result_reset; + +test_set "message" text: +From: stephan at example.org +Subject: frop +Message-ID: <432df324 at example.org> +To: nico at frop.example.org + +Frop +. +; + +test_set "envelope.to" "nico at frop.example.org"; + +test "NULL Sender" { + set "message" "I am not in today!"; + set "subject" "Out of office"; + set "from" "user at example.com"; + + vacation :from "${from}" :subject "${subject}" "${message}"; + + if not test_result_execute { + test_fail "execution of result failed"; + } + + test_message :smtp 0; + + if not envelope :is "from" "" { + if envelope :matches "from" "*" {} + test_fail "envelope sender not set properly: ${1}"; + } +} + +/* + * Send from recipient + */ + +test_result_reset; + +test_set "message" text: +From: stephan at example.org +Subject: frop +Message-ID: <432df324 at example.org> +To: nico at frop.example.org + +Frop +. +; + +test_set "envelope.to" "nico at frop.example.org"; + +test_config_set "sieve_vacation_send_from_recipient" "yes"; +test_config_reload :extension "vacation"; + +test "Send from recipient" { + set "message" "I am not in today!"; + set "subject" "Out of office"; + set "from" "user at example.com"; + + vacation :from "${from}" :subject "${subject}" "${message}"; + + if not test_result_execute { + test_fail "execution of result failed"; + } + + test_message :smtp 0; + + if not envelope "from" "nico at frop.example.org" { + test_fail "envelope sender not set properly"; + } +} + From dovecot at dovecot.org Wed Sep 25 10:11:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 25 Sep 2013 10:11:14 +0300 Subject: dovecot-2.2: Released v2.2.6. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1de9a494cb25 changeset: 16824:1de9a494cb25 user: Timo Sirainen date: Wed Sep 25 10:06:08 2013 +0300 description: Released v2.2.6. diffstat: NEWS | 21 +++++++++++++++++++++ TODO | 3 --- configure.ac | 4 ++-- 3 files changed, 23 insertions(+), 5 deletions(-) diffs (54 lines): diff -r a991a0547daa -r 1de9a494cb25 NEWS --- a/NEWS Mon Sep 23 04:25:16 2013 +0300 +++ b/NEWS Wed Sep 25 10:06:08 2013 +0300 @@ -1,3 +1,24 @@ +v2.2.6 2013-09-25 Timo Sirainen + + * acl: If public/shared namespace has a shared subscriptions file for + all users, don't list subscription entries that are not visible to + the user accessing it. + + + doveadm: Added "auth lookup" command for doing passdb lookup. + + login_log_format_elements: Added %{orig_user}, %{orig_username} + and %{orig_domain} expanding to the username exactly as sent by + the client (before any changes auth process made). + + Added ssl_prefer_server_ciphers setting. + + auth_verbose_passwords: Log the password also for unknown users. + + Linux: Added optional support for SO_REUSEPORT with + inet_listener { reuse_port=yes } + - director: v2.2.5 changes caused "SYNC lost" errors + - dsync: Many fixes and error handling improvements + - doveadm -A: Don't waste CPU by doing a separate config lookup + for each user + - Long-running ssl-params process no longer prevents Dovecot restart + - mbox: Fixed mailbox_list_index=yes to work correctly + v2.2.5 2013-08-05 Timo Sirainen + SSL: Added support for ECDH/ECDHE cipher suites (by David Hicks) diff -r a991a0547daa -r 1de9a494cb25 TODO --- a/TODO Mon Sep 23 04:25:16 2013 +0300 +++ b/TODO Wed Sep 25 10:06:08 2013 +0300 @@ -1,8 +1,5 @@ - dsync: delete foo, rename bar foo -> foo, foo-temp-1 - dsync+imapc: - - half-sync [-1], which is the same as normal sync except the other side - doesn't do anything (so when migrating from imapc, don't push changes - back to it) - mailbox list could be synced pretty optimally by ignoring (name, uidvalidity) matches. for the left if uidvalidities are unique and can be matched -> rename mailbox. diff -r a991a0547daa -r 1de9a494cb25 configure.ac --- a/configure.ac Mon Sep 23 04:25:16 2013 +0300 +++ b/configure.ac Wed Sep 25 10:06:08 2013 +0300 @@ -2,8 +2,8 @@ # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.5],[dovecot at dovecot.org]) -AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv5($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_INIT([Dovecot],[2.2.6],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv6($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) From dovecot at dovecot.org Wed Sep 25 10:11:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 25 Sep 2013 10:11:14 +0300 Subject: dovecot-2.2: Added tag 2.2.6 for changeset 1de9a494cb25 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa7d4f1b8fb8 changeset: 16825:fa7d4f1b8fb8 user: Timo Sirainen date: Wed Sep 25 10:06:08 2013 +0300 description: Added tag 2.2.6 for changeset 1de9a494cb25 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 1de9a494cb25 -r fa7d4f1b8fb8 .hgtags --- a/.hgtags Wed Sep 25 10:06:08 2013 +0300 +++ b/.hgtags Wed Sep 25 10:06:08 2013 +0300 @@ -109,3 +109,4 @@ 5d9f52c9a2871245097c725b9a4163a8789512fd 2.2.3 4b3c9c3e4fb8499ca7133a4d229e5eef349995e0 2.2.4 71f0696749ab347119278d7f68a01f81ed584c53 2.2.5 +1de9a494cb2520986da97fa73cf87a3781ea2d17 2.2.6 From dovecot at dovecot.org Wed Sep 25 10:11:14 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 25 Sep 2013 10:11:14 +0300 Subject: dovecot-2.2: Added signature for changeset 1de9a494cb25 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f84a1e3c4aa changeset: 16826:0f84a1e3c4aa user: Timo Sirainen date: Wed Sep 25 10:06:19 2013 +0300 description: Added signature for changeset 1de9a494cb25 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r fa7d4f1b8fb8 -r 0f84a1e3c4aa .hgsigs --- a/.hgsigs Wed Sep 25 10:06:08 2013 +0300 +++ b/.hgsigs Wed Sep 25 10:06:19 2013 +0300 @@ -72,3 +72,4 @@ 5d9f52c9a2871245097c725b9a4163a8789512fd 0 iEYEABECAAYFAlG+LqYACgkQyUhSUUBVisn45wCgiEnDSrMoa5hZHRv+0eovGQJJ4g8An1Q8kms24rovLF/PGaani8Ap6VXR 4b3c9c3e4fb8499ca7133a4d229e5eef349995e0 0 iEYEABECAAYFAlHI1NMACgkQyUhSUUBVismCOQCbB/qdjOfUBvm0AUGBudvg2TaKbJ0AnRmr8JEV1QnowwA0g0vGF8Hkm6ib 71f0696749ab347119278d7f68a01f81ed584c53 0 iEYEABECAAYFAlH/+4cACgkQyUhSUUBVislEUwCfc7GeL9I5US5yJltYpWe8ZU7MAIkAnRlUGvhpFMBnehwSlUDRsssRCJ+a +1de9a494cb2520986da97fa73cf87a3781ea2d17 0 iEYEABECAAYFAlJCi2cACgkQyUhSUUBVisnoWQCfecKzvsHw/MC/5DDuZXIF/CmkaBUAn2DfADK7V0fIWZ49GPyL+BPbYCXy From dovecot at dovecot.org Thu Sep 26 05:56:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 26 Sep 2013 05:56:18 +0300 Subject: dovecot-2.2: lib-http: 32bit system fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83e74b3a0d10 changeset: 16827:83e74b3a0d10 user: Timo Sirainen date: Thu Sep 26 04:55:59 2013 +0200 description: lib-http: 32bit system fix. diffstat: src/lib-http/http-header-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 0f84a1e3c4aa -r 83e74b3a0d10 src/lib-http/http-header-parser.c --- a/src/lib-http/http-header-parser.c Wed Sep 25 10:06:19 2013 +0300 +++ b/src/lib-http/http-header-parser.c Thu Sep 26 04:55:59 2013 +0200 @@ -272,7 +272,7 @@ const uoff_t max_size = parser->limits.max_size; const uoff_t max_field_size = parser->limits.max_field_size; const unsigned char *data; - uoff_t size; + size_t size; int ret; *error_r = NULL; From dovecot at dovecot.org Thu Sep 26 06:51:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 26 Sep 2013 06:51:54 +0300 Subject: dovecot-2.2: Panic if io_remove() fails with EBADF. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86e59823d0aa changeset: 16828:86e59823d0aa user: Timo Sirainen date: Thu Sep 26 05:51:43 2013 +0200 description: Panic if io_remove() fails with EBADF. It's a bug that shouldn't really be ignored. Panicing will give a debuggable core dump. diffstat: src/lib/ioloop-epoll.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (18 lines): diff -r 83e74b3a0d10 -r 86e59823d0aa src/lib/ioloop-epoll.c --- a/src/lib/ioloop-epoll.c Thu Sep 26 04:55:59 2013 +0200 +++ b/src/lib/ioloop-epoll.c Thu Sep 26 05:51:43 2013 +0200 @@ -143,8 +143,13 @@ op = last ? EPOLL_CTL_DEL : EPOLL_CTL_MOD; if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) { - i_error("epoll_ctl(%s, %d) failed: %m", + const char *errstr = t_strdup_printf( + "epoll_ctl(%s, %d) failed: %m", op == EPOLL_CTL_DEL ? "del" : "mod", io->fd); + if (errno == EBADF) + i_panic("%s", errstr); + else + i_error("%s", errstr); } } if (last) { From pigeonhole at rename-it.nl Thu Sep 26 22:40:50 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:40:50 +0200 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.2 for changeset 2176d400eca4 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/25ae2f39df94 changeset: 1809:25ae2f39df94 user: Stephan Bosch date: Thu Sep 26 21:40:34 2013 +0200 description: Added tag 0.4.2 for changeset 2176d400eca4 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 2176d400eca4 -r 25ae2f39df94 .hgtags --- a/.hgtags Thu Sep 26 21:40:22 2013 +0200 +++ b/.hgtags Thu Sep 26 21:40:34 2013 +0200 @@ -17,3 +17,4 @@ 374ec93999588b1acc554f81f61c93bf9fad7037 0.3.5 3a8dc1250e9b850044acbcd9d63d63597e67c7cb 0.4.0 f1535e2255cd84849d8b4fad6d82bdd6e01f810f 0.4.1 +2176d400eca4fa5d89597793a09c0b194eaed05f 0.4.2 From pigeonhole at rename-it.nl Thu Sep 26 22:40:50 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:40:50 +0200 Subject: dovecot-2.2-pigeonhole: Released v0.4.2 for Dovecot v2.2.6. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/2176d400eca4 changeset: 1808:2176d400eca4 user: Stephan Bosch date: Thu Sep 26 21:40:22 2013 +0200 description: Released v0.4.2 for Dovecot v2.2.6. diffstat: NEWS | 21 +++++++++++++++++++++ configure.ac | 2 +- 2 files changed, 22 insertions(+), 1 deletions(-) diffs (37 lines): diff -r 4924724b7f4f -r 2176d400eca4 NEWS --- a/NEWS Mon Sep 23 22:01:38 2013 +0200 +++ b/NEWS Thu Sep 26 21:40:22 2013 +0200 @@ -1,3 +1,24 @@ +v0.4.2 26-09-2013 Stephan Bosch + + * Incompatible change in Sieve doveadm plugin: the root attribute for + Sieve scripts is changed. Make sure that you update both sides of a dsync + setup simultaneously when Sieve is involved, otherwise synchronization will + likely fail. + + Added support for sending Sieve vacation replies with an actual sender, + rather than the default <> sender. Check the updated + doc/extensions/vacation.txt for more information. + - Fixed a binary code read problem in the `set' command of the Sieve variables + extension. Using the set command with a modifier and an empty string value + would cause code corruption problems while running the script. + - Various fixes for doveadm-sieve plugin, mostly crashes. These include a fix + for the `Invalid value for default sieve attribute' problem. + - Various fixes for compiler and static analyzer warnings, e.g. as reported + by CLang and on 32 bit systems. + - Fixed the implementation of the new :options flag for the Sieve include + extension. + - Fixed potential segfault bug at deinitialization of the lda-sieve plugin. + - Fixed messed up hex output for sieve-dump tool. + v0.4.1 03-06-2013 Stephan Bosch + Added support for handling temporary failures. These are passed back to diff -r 4924724b7f4f -r 2176d400eca4 configure.ac --- a/configure.ac Mon Sep 23 22:01:38 2013 +0200 +++ b/configure.ac Thu Sep 26 21:40:22 2013 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.1], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.2], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Thu Sep 26 22:42:26 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:42:26 +0200 Subject: dovecot-2.1-pigeonhole: Released v0.3.6 for Dovecot v2.1.17. Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/87d8f0aae957 changeset: 1700:87d8f0aae957 user: Stephan Bosch date: Thu Sep 26 21:41:24 2013 +0200 description: Released v0.3.6 for Dovecot v2.1.17. diffstat: NEWS | 12 ++++++++++++ configure.in | 2 +- 2 files changed, 13 insertions(+), 1 deletions(-) diffs (28 lines): diff -r aadc77880254 -r 87d8f0aae957 NEWS --- a/NEWS Mon Sep 16 00:55:29 2013 +0200 +++ b/NEWS Thu Sep 26 21:41:24 2013 +0200 @@ -1,3 +1,15 @@ +v0.3.6 26-09-2013 Stephan Bosch + + - Fixed a binary code read problem in the `set' command of the Sieve variables + extension. Using the set command with a modifier and an empty string value + would cause code corruption problems while running the script. + - Various fixes for compiler and static analyzer warnings, as reported + by CLang. + - ManageSieve: Fixed '[' ']' stupidity for response codes (only happened + before login). + - Fixed setting name in example-config/conf.d/20-managesieve.conf. + - Fixed messed up hex output for sieve-dump tool. + v0.3.5 09-05-2013 Stephan Bosch - Sieve editheader extension: fixed interaction with the Sieve body extension. diff -r aadc77880254 -r 87d8f0aae957 configure.in --- a/configure.in Mon Sep 16 00:55:29 2013 +0200 +++ b/configure.in Thu Sep 26 21:41:24 2013 +0200 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.3.5], [dovecot at dovecot.org], [dovecot-2.1-pigeonhole]) +AC_INIT([Pigeonhole], [0.3.6], [dovecot at dovecot.org], [dovecot-2.1-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Thu Sep 26 22:42:27 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:42:27 +0200 Subject: dovecot-2.1-pigeonhole: Added tag 0.3.6 for changeset 87d8f0aae957 Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/b9f545cd2ec8 changeset: 1701:b9f545cd2ec8 user: Stephan Bosch date: Thu Sep 26 21:41:35 2013 +0200 description: Added tag 0.3.6 for changeset 87d8f0aae957 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 87d8f0aae957 -r b9f545cd2ec8 .hgtags --- a/.hgtags Thu Sep 26 21:41:24 2013 +0200 +++ b/.hgtags Thu Sep 26 21:41:35 2013 +0200 @@ -15,3 +15,4 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0.3.3 4932026768454443d87d2e6445552b331589dbb1 0.3.4 374ec93999588b1acc554f81f61c93bf9fad7037 0.3.5 +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0.3.6 From pigeonhole at rename-it.nl Thu Sep 26 22:43:17 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:43:17 +0200 Subject: dovecot-2.2-pigeonhole: Added signature for changeset 2176d400eca4 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8159c26d0a36 changeset: 1810:8159c26d0a36 user: Stephan Bosch date: Thu Sep 26 21:43:09 2013 +0200 description: Added signature for changeset 2176d400eca4 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 25ae2f39df94 -r 8159c26d0a36 .hgsigs --- a/.hgsigs Thu Sep 26 21:40:34 2013 +0200 +++ b/.hgsigs Thu Sep 26 21:43:09 2013 +0200 @@ -11,3 +11,4 @@ 374ec93999588b1acc554f81f61c93bf9fad7037 0 iQEcBAABAgAGBQJRi6LgAAoJEATWKx49+7T0OPgH+gPOOYqdODGONhZTwxjKmvbVNzb428t/fS/pi2bSIeOXVzLone/P7NYstdM67eQsfxY4Kn8qN/eFkjdkozbc+3xx0VFvptFTd9NXGM57O84umgrvKXQDQM333TVMxMG5u6TPPkCmFcWmggcc1IdQxZ66dpgfETdh+IgJXdLQg4oc0IqHtqx19dVyYXRbtgY+GfA9ovFqOkzcWoXgSHRMmUf2gCGWPc3jSZPVHd3ObRvrwn3NcWULK4WCt5QPc6fU53eDrEvkSu6C5U0IV5G3jakrUK0PYtQ3jvxXGm8J3XQ+ea79Bp+mG/21Woz30Ey6AlFbuAHobGesGafjPLDLSNs= 3a8dc1250e9b850044acbcd9d63d63597e67c7cb 0 iQEcBAABAgAGBQJRi6eKAAoJEATWKx49+7T0+3sIAKj87rGC9kakJPL4RyFDc762HueCez8mUTRhe62K9+2LzVJL+2inGPY33PL5F7xZd0H/pxIfNfk/WGA78q6BkOaJSlGtz+wb5H/iZzSc+6GRfzPavlXlENgOKtnmzf3AoEKc0iNnieB5oYJTkJ2GtbPv2GD4u/Y6w7eTISfh/HlJeGvybR3L4XT6scNCr90/iQWjoZMNZSbYvKInSmrN9SGgtg0exqFfdkFmMahhSO/PGBq4hi5ZyFKqsnYEZpkLOduKW86nsn1GZiPD5/bEUD7lxJeeoRk3aobZoMQUlz4VRma6ag0MPRLpmzqUQFhpvDzpdg3dZSw/8NUQBa78w1M= f1535e2255cd84849d8b4fad6d82bdd6e01f810f 0 iQEcBAABAgAGBQJR1IRdAAoJEATWKx49+7T0uKMIANauBvbSO72bhrg8EI3Cvr2qy5nZrgqdgtZI0ZtIHbRBa9aBmxbX5lGqjWNLZaXwLc9dk7aeDoZds73kjiCYB+ac3UaXBaRscKNArkigTKFE0L2UZ5X5T/76iJ6diPFs8Eo/4ajYERDa3UQgUqHJ/8Fe2884bFDXMRmjbtVu2ceCw8Dlkee4kiVfjzuPRWud2jjtXcCoJXkFIQSbGcKsjlhd2yAGToJ9iw06tPNAFjupht9/wozyaZiZqJj7AnNvflHGMO1FDhOQJYxjz9JYA61h9F1vytGwfm1n3BVANpWSBJtkkuM/Tjp9GxeDB2jNrUbMikicSUceNtRLg9i0b+A= +2176d400eca4fa5d89597793a09c0b194eaed05f 0 iQEcBAABAgAGBQJSRI5JAAoJEATWKx49+7T0kGcIAKNHvRh/4idzHekHk5L81e0nu1IZRLyXg6YLkviF52ZYsB4WSbsiVhVYGoVS0XdSCvouQFq//In5S8nkUaSc1r4/2VK/vEchisYxwnF0YyLz5o7CaixLBCARLGGbjZTCyHvqOCHj6sRTJtc7GP1cdFXzYo7yIGg+W1hZn7yBB32p6stE+9SurxinK312LGFvBRrJ9t7y9gIeP1aV9yd0AhTorAP0XsOMflNQnnxsNXR/8LhwsqpOt9TR8hTVdGZHi2ZqjqK37/XAGYAJpBvpQRFq2UPWj6fEGNRLsh9TzGodQh/RoESvTeoHpEVy5o2yHUcnY71bYfMpb2IJH7NHKAA= From pigeonhole at rename-it.nl Thu Sep 26 22:44:43 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 21:44:43 +0200 Subject: dovecot-2.1-pigeonhole: Added signature for changeset 87d8f0aae957 Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/7b09c4608d42 changeset: 1702:7b09c4608d42 user: Stephan Bosch date: Thu Sep 26 21:44:37 2013 +0200 description: Added signature for changeset 87d8f0aae957 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r b9f545cd2ec8 -r 7b09c4608d42 .hgsigs --- a/.hgsigs Thu Sep 26 21:41:35 2013 +0200 +++ b/.hgsigs Thu Sep 26 21:44:37 2013 +0200 @@ -9,3 +9,4 @@ 64474c35967852bc452f71bc099ec3f8ded0369a 0 iQEcBAABAgAGBQJQWMvhAAoJEATWKx49+7T0NrYH/2PQuuFqzlku+NG8Iw0UN2yeDEML+2n1xG31ud7m3sNWw8lX+03gEd+LU8+LygHJJ0IAde/jBYRBbC8zj9UXDl3v5FIRwhcvGnllBCMMH7motfg+aLrCR/xs+0jV/AqpRin1VILHYFaB9UFP5PUgvJJiCUniQWoe+r41gra1hRA7OK3923YOOi9t4zJxoat7e0OMhc0IcdB7n3iQmyicbb8izKw/UvR2tR3T7fVcEl6u1LlbGaojtJA03V1L+a8QkmltiurD9VNmiHz++bGGJlA7LSmVYBq7BeC1lDnXUGO9ryZgln6aXRwUS0VaTI51F9gSMw+0UDJCwA5yBKqYyR8= 4932026768454443d87d2e6445552b331589dbb1 0 iQEcBAABAgAGBQJRYIePAAoJEATWKx49+7T0p2oH/ROhjt/m+wZmT9+2NxEDwnaOoQ8m9TxkZiZ50mmi/k4L7OQe/xffxM2T3NTTgRkaLCK2+MEz0pSLJXL+n/AjTBtiynwSxYY+W0wtYKBIs0tcQHaSafN2u5LjtQZ2RHg+Fi1szhJQu/jy31w12KGTqdyVw05JuEPgyfM7fKpqKh8CQZJucEyn9Vf8boGPQMFJ7N6o4wpOeW8RuVcBAEToHMpkDI1OQmB22cEQJfZrdOMPaucUOG6Abfw0FDwwV4bHa1nmxiPZXF8DpW31SVDoZgyvi08TSWFHS9t8Pij+XyWIdXhbCzqdpkvLGEsvMJjhiro9agf2tSj9jP+D4xiFRe8= 374ec93999588b1acc554f81f61c93bf9fad7037 0 iQEcBAABAgAGBQJRi6LgAAoJEATWKx49+7T0OPgH+gPOOYqdODGONhZTwxjKmvbVNzb428t/fS/pi2bSIeOXVzLone/P7NYstdM67eQsfxY4Kn8qN/eFkjdkozbc+3xx0VFvptFTd9NXGM57O84umgrvKXQDQM333TVMxMG5u6TPPkCmFcWmggcc1IdQxZ66dpgfETdh+IgJXdLQg4oc0IqHtqx19dVyYXRbtgY+GfA9ovFqOkzcWoXgSHRMmUf2gCGWPc3jSZPVHd3ObRvrwn3NcWULK4WCt5QPc6fU53eDrEvkSu6C5U0IV5G3jakrUK0PYtQ3jvxXGm8J3XQ+ea79Bp+mG/21Woz30Ey6AlFbuAHobGesGafjPLDLSNs= +87d8f0aae957ad290ea2e10033ce9ef70b291bb4 0 iQEcBAABAgAGBQJSRI6hAAoJEATWKx49+7T0j/YH/1ugfus0Bj/9MhG788eaaYXuF1UjusxvckT5ufkruQLkY0xt5aeDVPOCwEEhgjR5mr4yHYYQUg5xl6XvgErPymmx2SvTzjDvATSNK+smVU375Ho57RGNV9NVO1sKKXN5fqjeURfpu+iMNbaEpP/j9oQ+nukL1fFh3NLEsVXBCJTmJPqvV14jIpEFAwgUXSpUDVaTN8iAo/NuhmpcjnxAUR01tkJEVHH0hxExDYwXjGb76wUzgG9cLLqyFTn5qB8E2b0qaqaLY1h/K0exOJ01yWOYMiefm30k7viwQCEM+ZyyZ7ff355Gqcy/gJJSD7+jdTY7tEHbtBZcoE/zI9CYuqw= From pigeonhole at rename-it.nl Thu Sep 26 23:26:57 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 22:26:57 +0200 Subject: dovecot-2.0-pigeonhole: lib-sieve: Fixed code block read bounds ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.0-pigeonhole/rev/5d60c933bc58 changeset: 1562:5d60c933bc58 user: Stephan Bosch date: Sun Sep 15 17:47:53 2013 +0200 description: lib-sieve: Fixed code block read bounds checking. diffstat: src/lib-sieve/sieve-binary-code.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r 27285d227485 -r 5d60c933bc58 src/lib-sieve/sieve-binary-code.c --- a/src/lib-sieve/sieve-binary-code.c Sun Sep 15 12:19:10 2013 +0200 +++ b/src/lib-sieve/sieve-binary-code.c Sun Sep 15 17:47:53 2013 +0200 @@ -214,7 +214,7 @@ ((const int8_t *) (&_code[*address])) #define ADDR_BYTES_LEFT(address) \ - ((_code_size) - (*address)) + ((*address) > _code_size ? 0 : ((_code_size) - (*address))) #define ADDR_JUMP(address, offset) \ (*address) += offset @@ -350,7 +350,7 @@ ADDR_CODE_READ(sblock); - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return FALSE; (*offset_r) = code = ADDR_DATA_AT(address); @@ -382,7 +382,7 @@ if ( objs->count == 1 ) return objs->objects; - if ( ADDR_BYTES_LEFT(address) <= 0 ) + if ( ADDR_BYTES_LEFT(address) == 0 ) return NULL; code = ADDR_DATA_AT(address); From pigeonhole at rename-it.nl Thu Sep 26 23:26:57 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 22:26:57 +0200 Subject: dovecot-2.0-pigeonhole: lib-sieve: variables extension: Fixed co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.0-pigeonhole/rev/ded0c5a467aa changeset: 1564:ded0c5a467aa user: Stephan Bosch date: Thu Sep 26 22:26:48 2013 +0200 description: lib-sieve: variables extension: Fixed code corruption bug in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diffs (31 lines): diff -r f1dbb8d227fe -r ded0c5a467aa src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Thu Sep 26 22:24:07 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Thu Sep 26 22:26:48 2013 +0200 @@ -316,17 +316,17 @@ str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; - - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { - value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; - break; - } + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( str_len(value) > 0 ) { if ( modf.def != NULL && modf.def->modify != NULL ) { if ( !modf.def->modify(value, &new_value) ) { value = NULL; From pigeonhole at rename-it.nl Thu Sep 26 23:26:57 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 26 Sep 2013 22:26:57 +0200 Subject: dovecot-2.0-pigeonhole: lib-sieve: variables extension: fixed da... Message-ID: details: http://hg.rename-it.nl/dovecot-2.0-pigeonhole/rev/f1dbb8d227fe changeset: 1563:f1dbb8d227fe user: Stephan Bosch date: Thu Sep 26 22:24:07 2013 +0200 description: lib-sieve: variables extension: fixed data stack problem in 'set' command. diffstat: src/lib-sieve/plugins/variables/cmd-set.c | 86 +++++++++++++++--------------- 1 files changed, 42 insertions(+), 44 deletions(-) diffs (104 lines): diff -r 5d60c933bc58 -r f1dbb8d227fe src/lib-sieve/plugins/variables/cmd-set.c --- a/src/lib-sieve/plugins/variables/cmd-set.c Sun Sep 15 17:47:53 2013 +0200 +++ b/src/lib-sieve/plugins/variables/cmd-set.c Thu Sep 26 22:24:07 2013 +0200 @@ -315,58 +315,56 @@ if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - T_BEGIN { - /* Apply modifiers if necessary (sorted during code generation already) */ - if ( str_len(value) > 0 ) { - for ( i = 0; i < mdfs; i++ ) { - string_t *new_value; - struct sieve_variables_modifier modf; - - if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + /* Apply modifiers if necessary (sorted during code generation already) */ + if ( str_len(value) > 0 ) { + for ( i = 0; i < mdfs; i++ ) { + string_t *new_value; + struct sieve_variables_modifier modf; + + if ( !ext_variables_opr_modifier_read(renv, address, &modf) ) { + value = NULL; + ret = SIEVE_EXEC_BIN_CORRUPT; + break; + } + + if ( modf.def != NULL && modf.def->modify != NULL ) { + if ( !modf.def->modify(value, &new_value) ) { value = NULL; - ret = SIEVE_EXEC_BIN_CORRUPT; + ret = SIEVE_EXEC_FAILURE; break; } - - if ( modf.def != NULL && modf.def->modify != NULL ) { - if ( !modf.def->modify(value, &new_value) ) { - value = NULL; - ret = SIEVE_EXEC_FAILURE; - break; - } - sieve_runtime_trace_here - (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", - sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); + sieve_runtime_trace_here + (renv, SIEVE_TRLVL_COMMANDS, "modify :%s \"%s\" => \"%s\"", + sieve_variables_modifier_name(&modf), str_c(value), str_c(new_value)); - value = new_value; - if ( value == NULL ) - break; + value = new_value; + if ( value == NULL ) + break; - /* Hold value within limits */ - if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) - str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); - } - } - } - - /* Actually assign the value if all is well */ - if ( value != NULL ) { - if ( !sieve_variable_assign(storage, var_index, value) ) - ret = SIEVE_EXEC_BIN_CORRUPT; - else { - if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { - const char *var_name, *var_id; - - (void)sieve_variable_get_identifier(storage, var_index, &var_name); - var_id = sieve_variable_get_varid(storage, var_index); - - sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", - var_name, var_id, str_c(value)); - } + /* Hold value within limits */ + if ( str_len(value) > EXT_VARIABLES_MAX_VARIABLE_SIZE ) + str_truncate(value, EXT_VARIABLES_MAX_VARIABLE_SIZE); } } - } T_END; + } + + /* Actually assign the value if all is well */ + if ( value != NULL ) { + if ( !sieve_variable_assign(storage, var_index, value) ) + ret = SIEVE_EXEC_BIN_CORRUPT; + else { + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + const char *var_name, *var_id; + + (void)sieve_variable_get_identifier(storage, var_index, &var_name); + var_id = sieve_variable_get_varid(storage, var_index); + + sieve_runtime_trace_here(renv, 0, "assign `%s' [%s] = \"%s\"", + var_name, var_id, str_c(value)); + } + } + } if ( ret <= 0 ) return ret; if ( value == NULL ) return SIEVE_EXEC_FAILURE; From dovecot at dovecot.org Fri Sep 27 06:33:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 27 Sep 2013 06:33:52 +0300 Subject: dovecot-2.1: Compiler warning fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/caa466370fff changeset: 14999:caa466370fff user: Timo Sirainen date: Fri Sep 27 05:33:43 2013 +0200 description: Compiler warning fix. diffstat: src/lib/json-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r dc631b8b1290 -r caa466370fff src/lib/json-parser.c --- a/src/lib/json-parser.c Fri Sep 20 04:20:22 2013 +0300 +++ b/src/lib/json-parser.c Fri Sep 27 05:33:43 2013 +0200 @@ -185,7 +185,7 @@ { unsigned int len = strlen(atom); - if (parser->end - parser->data < len) + if ((size_t)(parser->end - parser->data) < len) return -1; if (memcmp(parser->data, atom, len) != 0) return -1; From dovecot at dovecot.org Sat Sep 28 09:09:18 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 28 Sep 2013 09:09:18 +0300 Subject: dovecot-2.1: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c546b384b76c changeset: 15000:c546b384b76c user: Timo Sirainen date: Sat Sep 28 09:09:04 2013 +0300 description: Compiler warning fix diffstat: src/lib-storage/index/maildir/maildir-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r caa466370fff -r c546b384b76c src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Fri Sep 27 05:33:43 2013 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sat Sep 28 09:09:04 2013 +0300 @@ -225,7 +225,7 @@ } else if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) { /* the directory should be empty. we won't do anything until ctime changes. */ - } else if (st.st_atime < ioloop_time - interval) { + } else if (st.st_atime < (time_t)(ioloop_time - interval)) { /* time to scan */ (void)unlink_old_files(path, "", ioloop_time - MAILDIR_TMP_DELETE_SECS); From dovecot at dovecot.org Sun Sep 29 07:05:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 29 Sep 2013 07:05:27 +0300 Subject: dovecot-2.1: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/056ed90355ca changeset: 15001:056ed90355ca user: Timo Sirainen date: Sun Sep 29 07:05:12 2013 +0300 description: Compiler warning fix diffstat: src/lib-storage/index/dbox-common/dbox-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c546b384b76c -r 056ed90355ca src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Sat Sep 28 09:09:04 2013 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Sun Sep 29 07:05:12 2013 +0300 @@ -165,7 +165,7 @@ } else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { /* there haven't been any changes to this directory since we last checked it. */ - } else if (st.st_atime < ioloop_time - interval) { + } else if (st.st_atime < (time_t)(ioloop_time - interval)) { /* time to scan */ const char *prefix = mailbox_list_get_global_temp_prefix(list); From dovecot at dovecot.org Mon Sep 30 06:41:11 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Sep 2013 06:41:11 +0300 Subject: dovecot-2.1: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/49258067fcfb changeset: 15002:49258067fcfb user: Timo Sirainen date: Mon Sep 30 06:40:50 2013 +0300 description: Compiler warning fix diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 056ed90355ca -r 49258067fcfb src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Sun Sep 29 07:05:12 2013 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Mon Sep 30 06:40:50 2013 +0300 @@ -158,7 +158,7 @@ } else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { /* there haven't been any changes to this directory since we last checked it. */ - } else if (st.st_atime < ioloop_time - interval) { + } else if (st.st_atime < (time_t)(ioloop_time - interval)) { /* time to scan */ (void)unlink_old_files(map->path, DBOX_TEMP_FILE_PREFIX, ioloop_time - DBOX_TMP_DELETE_SECS); From dovecot at dovecot.org Mon Sep 30 15:28:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 30 Sep 2013 15:28:17 +0300 Subject: dovecot-2.2: doveadm fs delete: Added -R parameter for recursive... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/657832a67daa changeset: 16830:657832a67daa user: Timo Sirainen date: Mon Sep 30 15:28:04 2013 +0300 description: doveadm fs delete: Added -R parameter for recursive deletion. diffstat: src/doveadm/doveadm-fs.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 61 insertions(+), 1 deletions(-) diffs (98 lines): diff -r 17e21e20e6bb -r 657832a67daa src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Mon Sep 30 09:04:43 2013 +0300 +++ b/src/doveadm/doveadm-fs.c Mon Sep 30 15:28:04 2013 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "istream.h" #include "ostream.h" #include "iostream-ssl.h" @@ -10,6 +11,7 @@ #include static void fs_cmd_help(doveadm_command_t *cmd); +static void cmd_fs_delete(int argc, char *argv[]); static struct fs * cmd_fs_init(int *argc, char **argv[], int own_arg_count, doveadm_command_t *cmd) @@ -154,11 +156,69 @@ fs_deinit(&fs); } +static void cmd_fs_delete_dir_recursive(struct fs *fs, const char *path) +{ + struct fs_iter *iter; + struct fs_file *file; + ARRAY_TYPE(const_string) dirs; + const char *fname, *const *fnamep; + + /* delete subdirs first. all fs backends can't handle recursive + lookups, so save the list first. */ + t_array_init(&dirs, 8); + iter = fs_iter_init(fs, path, FS_ITER_FLAG_DIRS); + while ((fname = fs_iter_next(iter)) != NULL) { + fname = t_strdup(fname); + array_append(&dirs, &fname, 1); + } + if (fs_iter_deinit(&iter) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", + path, fs_last_error(fs)); + doveadm_exit_code = EX_TEMPFAIL; + } + array_foreach(&dirs, fnamep) T_BEGIN { + cmd_fs_delete_dir_recursive(fs, + t_strdup_printf("%s/%s", path, *fnamep)); + } T_END; + + /* delete files */ + iter = fs_iter_init(fs, path, 0); + while ((fname = fs_iter_next(iter)) != NULL) T_BEGIN { + file = fs_file_init(fs, t_strdup_printf("%s/%s", path, fname), + FS_OPEN_MODE_READONLY); + if (fs_delete(file) < 0) { + i_error("fs_delete(%s) failed: %s", + fs_file_path(file), fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + } T_END; + if (fs_iter_deinit(&iter) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", + path, fs_last_error(fs)); + doveadm_exit_code = EX_TEMPFAIL; + } +} + +static void cmd_fs_delete_recursive(int argc, char *argv[]) +{ + struct fs *fs; + + fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete); + cmd_fs_delete_dir_recursive(fs, argv[0]); + fs_deinit(&fs); +} + static void cmd_fs_delete(int argc, char *argv[]) { struct fs *fs; struct fs_file *file; + if (null_strcmp(argv[1], "-R") == 0) { + cmd_fs_delete_recursive(argc-1, argv+1); + return; + } + fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete); file = fs_file_init(fs, argv[0], FS_OPEN_MODE_READONLY); @@ -211,7 +271,7 @@ { cmd_fs_put, "fs put", " " }, { cmd_fs_copy, "fs copy", " " }, { cmd_fs_stat, "fs stat", " " }, - { cmd_fs_delete, "fs delete", " " }, + { cmd_fs_delete, "fs delete", "[-R] " }, { cmd_fs_iter, "fs iter", " " }, { cmd_fs_iter_dirs, "fs iter-dirs", " " }, };