From dovecot at dovecot.org Tue Oct 1 06:58:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 01 Oct 2013 06:58:15 +0300 Subject: dovecot-2.1: Compiler warning fixes. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/4b80f6be7ff8 changeset: 15003:4b80f6be7ff8 user: Timo Sirainen date: Tue Oct 01 06:57:54 2013 +0300 description: Compiler warning fixes. diffstat: src/director/user-directory.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (30 lines): diff -r 49258067fcfb -r 4b80f6be7ff8 src/director/user-directory.c --- a/src/director/user-directory.c Mon Sep 30 06:40:50 2013 +0300 +++ b/src/director/user-directory.c Tue Oct 01 06:57:54 2013 +0300 @@ -110,7 +110,7 @@ struct user *pos, struct user *user) { for (; pos != NULL; pos = pos->prev) { - if ((time_t)pos->timestamp <= user->timestamp) + if (pos->timestamp <= user->timestamp) break; } if (pos == NULL) @@ -131,7 +131,7 @@ struct user *pos, struct user *user) { for (; pos != NULL; pos = pos->next) { - if ((time_t)pos->timestamp >= user->timestamp) + if (pos->timestamp >= user->timestamp) break; } if (pos == NULL) @@ -174,7 +174,7 @@ if (dir->prev_insert_pos == NULL) { /* find the position starting from tail */ user_directory_insert_backwards(dir, dir->tail, user); - } else if (timestamp < dir->prev_insert_pos->timestamp) { + } else if (timestamp < (time_t)dir->prev_insert_pos->timestamp) { user_directory_insert_backwards(dir, dir->prev_insert_pos, user); } else { From dovecot at dovecot.org Wed Oct 2 08:02:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Oct 2013 08:02:52 +0300 Subject: dovecot-2.1: Compiler warning fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/9313b0733008 changeset: 15004:9313b0733008 user: Timo Sirainen date: Wed Oct 02 08:02:30 2013 +0300 description: Compiler warning fix. diffstat: src/director/director-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4b80f6be7ff8 -r 9313b0733008 src/director/director-connection.c --- a/src/director/director-connection.c Tue Oct 01 06:57:54 2013 +0300 +++ b/src/director/director-connection.c Wed Oct 02 08:02:30 2013 +0300 @@ -1175,7 +1175,7 @@ } if ((host == NULL || !host->self) && - dir->self_host->last_sync_timestamp != ioloop_time) + (time_t)dir->self_host->last_sync_timestamp != ioloop_time) director_resend_sync(dir); return TRUE; } From dovecot at dovecot.org Wed Oct 2 11:48:42 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 02 Oct 2013 11:48:42 +0300 Subject: dovecot-2.2: mailbox_list_index: Fixed assert-crash sometimes wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a9844ea66c83 changeset: 16831:a9844ea66c83 user: Timo Sirainen date: Wed Oct 02 11:48:31 2013 +0300 description: mailbox_list_index: Fixed assert-crash sometimes when deleting a mailbox. diffstat: src/lib-storage/list/mailbox-list-index.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 657832a67daa -r a9844ea66c83 src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Mon Sep 30 15:28:04 2013 +0300 +++ b/src/lib-storage/list/mailbox-list-index.c Wed Oct 02 11:48:31 2013 +0300 @@ -363,6 +363,9 @@ struct mail_index_view *view; int ret; + if (ilist->syncing) + return 0; + if (mailbox_list_index_index_open(list) < 0) return -1; if (mail_index_refresh(ilist->index) < 0) { From dovecot at dovecot.org Thu Oct 3 11:20:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Oct 2013 11:20:27 +0300 Subject: dovecot-2.2: dsync: Removed periodic commits while importing. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/81aedacbb01f changeset: 16832:81aedacbb01f user: Timo Sirainen date: Thu Oct 03 11:20:09 2013 +0300 description: dsync: Removed periodic commits while importing. The problem is that the current algorithm doesn't handle it correctly. If there already are some mails that exist locally, or remote has duplicate mails, the messages may not be saved in the UID order, and if there's a commit between such wrong ordering, the UIDs get renumbered. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 9 --------- 1 files changed, 0 insertions(+), 9 deletions(-) diffs (26 lines): diff -r a9844ea66c83 -r 81aedacbb01f src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Oct 02 11:48:31 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Oct 03 11:20:09 2013 +0300 @@ -15,8 +15,6 @@ #include "dsync-mailbox.h" #include "dsync-mailbox-import.h" -#define DSYNC_COMMIT_MSGS_INTERVAL 100 - struct importer_mail { const char *guid; uint32_t uid; @@ -1553,13 +1551,6 @@ if (importer->highest_wanted_uid < uid) importer->highest_wanted_uid = uid; array_append(&importer->wanted_uids, &uid, 1); - - /* commit the transaction once in a while, so if we fail we don't - rollback everything. */ - if (array_count(&importer->wanted_uids) % DSYNC_COMMIT_MSGS_INTERVAL == 0) { - if (dsync_mailbox_import_commit(importer, FALSE) < 0) - importer->failed = TRUE; - } } static bool From dovecot at dovecot.org Thu Oct 3 11:35:04 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Oct 2013 11:35:04 +0300 Subject: dovecot-2.2: If DEBUG_OUTOFMEM environment is set, abort() on "o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b5ec4d364c2e changeset: 16833:b5ec4d364c2e user: Timo Sirainen date: Thu Oct 03 11:34:48 2013 +0300 description: If DEBUG_OUTOFMEM environment is set, abort() on "out of memory" errors. diffstat: src/lib/failures.c | 3 ++- src/master/master-settings.c | 2 +- src/master/service-process.c | 19 +++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diffs (61 lines): diff -r 81aedacbb01f -r b5ec4d364c2e src/lib/failures.c --- a/src/lib/failures.c Thu Oct 03 11:20:09 2013 +0300 +++ b/src/lib/failures.c Thu Oct 03 11:34:48 2013 +0300 @@ -187,7 +187,8 @@ i_error("Raw backtrace: %s", backtrace); } - if (type == LOG_TYPE_PANIC) + if (type == LOG_TYPE_PANIC || + (status == FATAL_OUTOFMEM && getenv("DEBUG_OUTOFMEM") != NULL)) abort(); else failure_exit(status); diff -r 81aedacbb01f -r b5ec4d364c2e src/master/master-settings.c --- a/src/master/master-settings.c Thu Oct 03 11:20:09 2013 +0300 +++ b/src/master/master-settings.c Thu Oct 03 11:34:48 2013 +0300 @@ -217,7 +217,7 @@ .state_dir = PKG_STATEDIR, .libexec_dir = PKG_LIBEXECDIR, .instance_name = PACKAGE, - .import_environment = "TZ" ENV_SYSTEMD ENV_GDB, + .import_environment = "TZ DEBUG_OUTOFMEM" ENV_SYSTEMD ENV_GDB, .protocols = "imap pop3 lmtp", .listen = "*, ::", .ssl = "yes:no:required", diff -r 81aedacbb01f -r b5ec4d364c2e src/master/service-process.c --- a/src/master/service-process.c Thu Oct 03 11:20:09 2013 +0300 +++ b/src/master/service-process.c Thu Oct 03 11:34:48 2013 +0300 @@ -405,6 +405,8 @@ static const char * get_exit_status_message(struct service *service, enum fatal_exit_status status) { + string_t *str; + switch (status) { case FATAL_LOGOPEN: return "Can't open log file"; @@ -413,12 +415,17 @@ case FATAL_LOGERROR: return "Internal logging error"; case FATAL_OUTOFMEM: - if (service->vsz_limit == 0) - return "Out of memory"; - return t_strdup_printf("Out of memory (service %s { vsz_limit=%u MB }, " - "you may need to increase it)", - service->set->name, - (unsigned int)(service->vsz_limit/1024/1024)); + str = t_str_new(128); + str_append(str, "Out of memory"); + if (service->vsz_limit != 0) { + str_printfa(str, " (service %s { vsz_limit=%u MB }, " + "you may need to increase it)", + service->set->name, + (unsigned int)(service->vsz_limit/1024/1024)); + } + if (getenv("DEBUG_OUTOFMEM") == NULL) + str_append(str, " - set DEBUG_OUTOFMEM=1 environment to get core dump"); + return str_c(str); case FATAL_EXEC: return "exec() failed"; From dovecot at dovecot.org Thu Oct 3 14:57:23 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 03 Oct 2013 14:57:23 +0300 Subject: dovecot-2.2: dsync: Added missing alarm(0) Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/74b3f504d797 changeset: 16834:74b3f504d797 user: Timo Sirainen date: Thu Oct 03 14:56:23 2013 +0300 description: dsync: Added missing alarm(0) This didn't really cause any problems normally since the process was usually just about to die anyway. diffstat: src/doveadm/dsync/doveadm-dsync.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b5ec4d364c2e -r 74b3f504d797 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Thu Oct 03 11:34:48 2013 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Thu Oct 03 14:56:23 2013 +0300 @@ -405,6 +405,7 @@ } *status_r = -1; } + alarm(0); } static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged) From dovecot at dovecot.org Sat Oct 5 13:29:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 05 Oct 2013 13:29:54 +0300 Subject: dovecot-2.2: lib-fs: Added FS_OPEN_FLAG_ASYNC_NOQUEUE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5672c40cf421 changeset: 16835:5672c40cf421 user: Timo Sirainen date: Sat Oct 05 13:29:26 2013 +0300 description: lib-fs: Added FS_OPEN_FLAG_ASYNC_NOQUEUE diffstat: src/lib-fs/fs-api.c | 2 ++ src/lib-fs/fs-api.h | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (29 lines): diff -r 74b3f504d797 -r 5672c40cf421 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Oct 03 14:56:23 2013 +0300 +++ b/src/lib-fs/fs-api.c Sat Oct 05 13:29:26 2013 +0300 @@ -159,6 +159,8 @@ struct fs_file *file; i_assert(path != NULL); + i_assert((mode_flags & FS_OPEN_FLAG_ASYNC_NOQUEUE) == 0 || + (mode_flags & FS_OPEN_FLAG_ASYNC) != 0); T_BEGIN { file = fs->v.file_init(fs, path, mode_flags & FS_OPEN_MODE_MASK, diff -r 74b3f504d797 -r 5672c40cf421 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Thu Oct 03 14:56:23 2013 +0300 +++ b/src/lib-fs/fs-api.h Sat Oct 05 13:29:26 2013 +0300 @@ -54,7 +54,12 @@ finished and fs_read_stream() returns a nonblocking stream. */ FS_OPEN_FLAG_ASYNC = 0x20, /* fs_read_stream() must return a seekable input stream */ - FS_OPEN_FLAG_SEEKABLE = 0x40 + FS_OPEN_FLAG_SEEKABLE = 0x40, + /* Backend should handle this file's operations immediately without + any additional command queueing. The caller is assumed to be the one + doing any rate limiting if needed. This flag can only be used with + ASYNC flag, synchronous requests are never queued. */ + FS_OPEN_FLAG_ASYNC_NOQUEUE = 0x80 }; enum fs_iter_flags { From dovecot at dovecot.org Sun Oct 6 13:21:26 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 06 Oct 2013 13:21:26 +0300 Subject: dovecot-2.2: lib-http: Fixed linking with some compilers Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e44453887b6 changeset: 16836:4e44453887b6 user: Timo Sirainen date: Sun Oct 06 13:21:15 2013 +0300 description: lib-http: Fixed linking with some compilers Inline functions in http-header.h still required linking http-header.lo with e.g. Sun Studio diffstat: src/lib-http/Makefile.am | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (29 lines): diff -r 5672c40cf421 -r 4e44453887b6 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Sat Oct 05 13:29:26 2013 +0300 +++ b/src/lib-http/Makefile.am Sun Oct 06 13:21:15 2013 +0300 @@ -67,7 +67,7 @@ ../lib/liblib.la test_http_url_SOURCES = test-http-url.c -test_http_url_LDADD = http-url.lo $(test_libs) +test_http_url_LDADD = http-url.lo http-header.lo $(test_libs) test_http_url_DEPENDENCIES = $(test_deps) test_http_date_SOURCES = test-http-date.c @@ -75,7 +75,7 @@ test_http_date_DEPENDENCIES = $(test_deps) test_http_header_parser_SOURCES = test-http-header-parser.c -test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo $(test_libs) +test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo http-header.lo $(test_libs) test_http_header_parser_DEPENDENCIES = $(test_deps) test_http_transfer_SOURCES = test-http-transfer.c @@ -83,6 +83,7 @@ http-parser.lo \ http-header-parser.lo \ http-transfer-chunked.lo \ + http-header.lo \ $(test_libs) test_http_transfer_DEPENDENCIES = $(test_deps) From pigeonhole at rename-it.nl Sun Oct 6 23:14:12 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 06 Oct 2013 22:14:12 +0200 Subject: dovecot-2.2-pigeonhole: Sieve extprograms plugin: Forgot to disa... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/689dbc10a7fc changeset: 1811:689dbc10a7fc user: Stephan Bosch date: Sun Oct 06 22:14:01 2013 +0200 description: Sieve extprograms plugin: Forgot to disable the alarm() timeouts set for script execution. This may cause later syscalls to exit with EINTR unexpectedly. diffstat: src/plugins/sieve-extprograms/script-client-local.c | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diffs (47 lines): diff -r 8159c26d0a36 -r 689dbc10a7fc src/plugins/sieve-extprograms/script-client-local.c --- a/src/plugins/sieve-extprograms/script-client-local.c Thu Sep 26 21:43:09 2013 +0200 +++ b/src/plugins/sieve-extprograms/script-client-local.c Sun Oct 06 22:14:01 2013 +0200 @@ -148,7 +148,7 @@ { struct script_client_local *slclient = (struct script_client_local *) sclient; - pid_t pid = slclient->pid; + pid_t pid = slclient->pid, ret; time_t runtime, timeout = 0; int status; @@ -169,9 +169,12 @@ /* Wait for child to exit */ force = force || (timeout == 0 && sclient->set->input_idle_timeout_secs > 0); - if ( !force ) + if ( !force ) { alarm(timeout); - if ( force || waitpid(pid, &status, 0) < 0 ) { + ret = waitpid(pid, &status, 0); + alarm(0); + } + if ( force || ret < 0 ) { if ( !force && errno != EINTR ) { i_error("waitpid(%s) failed: %m", sclient->path); (void)kill(pid, SIGKILL); @@ -197,7 +200,9 @@ /* Wait for it to die (give it some more time) */ alarm(5); - if ( waitpid(pid, &status, 0) < 0 ) { + ret = waitpid(pid, &status, 0); + alarm(0); + if ( ret < 0 ) { if ( errno != EINTR ) { i_error("waitpid(%s) failed: %m", sclient->path); (void)kill(pid, SIGKILL); @@ -223,7 +228,7 @@ return -1; } } - } + } /* Evaluate child exit status */ sclient->exit_code = -1; From dovecot at dovecot.org Tue Oct 8 10:05:01 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Oct 2013 10:05:01 +0300 Subject: dovecot-2.2: lib-http: Added setting for User-Agent header. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fe009d4ba4ed changeset: 16837:fe009d4ba4ed user: Timo Sirainen date: Tue Oct 08 10:04:55 2013 +0300 description: lib-http: Added setting for User-Agent header. diffstat: src/lib-http/http-client-private.h | 1 + src/lib-http/http-client-request.c | 8 ++++++++ src/lib-http/http-client.c | 10 ++++------ src/lib-http/http-client.h | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) diffs (69 lines): diff -r 4e44453887b6 -r fe009d4ba4ed src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sun Oct 06 13:21:15 2013 +0300 +++ b/src/lib-http/http-client-private.h Tue Oct 08 10:04:55 2013 +0300 @@ -74,6 +74,7 @@ unsigned int have_hdr_expect:1; unsigned int have_hdr_host:1; unsigned int have_hdr_body_spec:1; + unsigned int have_hdr_user_agent:1; unsigned int payload_sync:1; unsigned int payload_chunked:1; diff -r 4e44453887b6 -r fe009d4ba4ed src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sun Oct 06 13:21:15 2013 +0300 +++ b/src/lib-http/http-client-request.c Tue Oct 08 10:04:55 2013 +0300 @@ -175,6 +175,10 @@ if (strcasecmp(key, "Transfer-Encoding") == 0) req->have_hdr_body_spec = TRUE; break; + case 'u': case 'U': + if (strcasecmp(key, "User-Agent") == 0) + req->have_hdr_user_agent = TRUE; + break; } str_printfa(req->headers, "%s: %s\r\n", key, value); } @@ -450,6 +454,10 @@ str_append(rtext, http_date_create(req->date)); str_append(rtext, "\r\n"); } + if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) { + str_printfa(rtext, "User-Agent: %s\r\n", + req->client->set.user_agent); + } if (!req->have_hdr_expect && req->payload_sync) { str_append(rtext, "Expect: 100-continue\r\n"); } diff -r 4e44453887b6 -r fe009d4ba4ed src/lib-http/http-client.c --- a/src/lib-http/http-client.c Sun Oct 06 13:21:15 2013 +0300 +++ b/src/lib-http/http-client.c Tue Oct 08 10:04:55 2013 +0300 @@ -75,12 +75,10 @@ pool = pool_alloconly_create("http client", 1024); client = p_new(pool, struct http_client, 1); client->pool = pool; - if (set->dns_client_socket_path != NULL && *set->dns_client_socket_path != '\0') { - client->set.dns_client_socket_path = - p_strdup(pool, set->dns_client_socket_path); - } - if (set->rawlog_dir != NULL && *set->rawlog_dir != '\0') - client->set.rawlog_dir = p_strdup(pool, set->rawlog_dir); + client->set.dns_client_socket_path = + p_strdup_empty(pool, set->dns_client_socket_path); + client->set.user_agent = p_strdup_empty(pool, set->user_agent); + client->set.rawlog_dir = p_strdup_empty(pool, set->rawlog_dir); client->set.ssl_ca_dir = p_strdup(pool, set->ssl_ca_dir); client->set.ssl_ca_file = p_strdup(pool, set->ssl_ca_file); client->set.ssl_ca = p_strdup(pool, set->ssl_ca); diff -r 4e44453887b6 -r fe009d4ba4ed src/lib-http/http-client.h --- a/src/lib-http/http-client.h Sun Oct 06 13:21:15 2013 +0300 +++ b/src/lib-http/http-client.h Tue Oct 08 10:04:55 2013 +0300 @@ -41,6 +41,9 @@ /* user cert */ const char *ssl_cert, *ssl_key, *ssl_key_password; + /* User-Agent: header (default: none) */ + const char *user_agent; + const char *rawlog_dir; unsigned int max_idle_time_msecs; From dovecot at dovecot.org Tue Oct 8 16:48:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 08 Oct 2013 16:48:16 +0300 Subject: dovecot-2.2: auth: Added ability to truncate values logged by au... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc46ae14008c changeset: 16838:dc46ae14008c user: Timo Sirainen date: Tue Oct 08 16:48:04 2013 +0300 description: auth: Added ability to truncate values logged by auth_verbose_passwords. diffstat: doc/example-config/conf.d/10-logging.conf | 1 + src/auth/auth-request.c | 15 +++++++++++-- src/auth/auth-settings.c | 33 +++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 5 deletions(-) diffs (109 lines): diff -r fe009d4ba4ed -r dc46ae14008c doc/example-config/conf.d/10-logging.conf --- a/doc/example-config/conf.d/10-logging.conf Tue Oct 08 10:04:55 2013 +0300 +++ b/doc/example-config/conf.d/10-logging.conf Tue Oct 08 16:48:04 2013 +0300 @@ -26,6 +26,7 @@ # In case of password mismatches, log the attempted password. Valid values are # no, plain and sha1. sha1 can be useful for detecting brute force password # attempts vs. user simply trying the same password over and over again. +# You can also truncate the value to n chars by appending ":n" (e.g. sha1:6). #auth_verbose_passwords = no # Even more verbose logging for debugging purposes. Shows for example SQL diff -r fe009d4ba4ed -r dc46ae14008c src/auth/auth-request.c --- a/src/auth/auth-request.c Tue Oct 08 10:04:55 2013 +0300 +++ b/src/auth/auth-request.c Tue Oct 08 16:48:04 2013 +0300 @@ -1785,18 +1785,27 @@ static void auth_request_append_password(struct auth_request *request, string_t *str) { - const char *log_type = request->set->verbose_passwords; + const char *p, *log_type = request->set->verbose_passwords; + unsigned int max_len = UINT_MAX; + + p = strchr(log_type, ':'); + if (p != NULL) { + if (str_to_uint(p+1, &max_len) < 0) + i_unreached(); + log_type = t_strdup_until(log_type, p); + } if (strcmp(log_type, "plain") == 0) { str_printfa(str, "(given password: %s)", - request->mech_password); + t_strndup(request->mech_password, max_len)); } else if (strcmp(log_type, "sha1") == 0) { unsigned char sha1[SHA1_RESULTLEN]; sha1_get_digest(request->mech_password, strlen(request->mech_password), sha1); str_printfa(str, "(SHA1 of given password: %s)", - binary_to_hex(sha1, sizeof(sha1))); + t_strndup(binary_to_hex(sha1, sizeof(sha1)), + max_len)); } else { i_unreached(); } diff -r fe009d4ba4ed -r dc46ae14008c src/auth/auth-settings.c --- a/src/auth/auth-settings.c Tue Oct 08 10:04:55 2013 +0300 +++ b/src/auth/auth-settings.c Tue Oct 08 16:48:04 2013 +0300 @@ -214,7 +214,7 @@ DEF(SET_BOOL, verbose), DEF(SET_BOOL, debug), DEF(SET_BOOL, debug_passwords), - DEF(SET_ENUM, verbose_passwords), + DEF(SET_STR, verbose_passwords), DEF(SET_BOOL, ssl_require_client_cert), DEF(SET_BOOL, ssl_username_from_cert), DEF(SET_BOOL, use_winbind), @@ -253,7 +253,7 @@ .verbose = FALSE, .debug = FALSE, .debug_passwords = FALSE, - .verbose_passwords = "no:plain:sha1", + .verbose_passwords = "no", .ssl_require_client_cert = FALSE, .ssl_username_from_cert = FALSE, .use_winbind = FALSE, @@ -314,6 +314,32 @@ return TRUE; } +static bool +auth_verify_verbose_password(const struct auth_settings *set, + const char **error_r) +{ + const char *p, *value = set->verbose_passwords; + unsigned int num; + + p = strchr(value, ':'); + if (p != NULL) { + if (str_to_uint(p+1, &num) < 0 || num == 0) { + *error_r = t_strdup_printf("auth_verbose_passwords: " + "Invalid truncation number: '%s'", p+1); + return FALSE; + } + value = t_strdup_until(value, p); + } + if (strcmp(value, "no") == 0) + return TRUE; + else if (strcmp(value, "plain") == 0) + return TRUE; + else if (strcmp(value, "sha1") == 0) + return TRUE; + else + return FALSE; +} + static bool auth_settings_check(void *_set, pool_t pool, const char **error_r) { @@ -339,6 +365,9 @@ return FALSE; } + if (!auth_verify_verbose_password(set, error_r)) + return FALSE; + if (*set->username_chars == '\0') { /* all chars are allowed */ memset(set->username_chars_map, 1, From dovecot at dovecot.org Thu Oct 10 20:50:28 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 10 Oct 2013 20:50:28 +0300 Subject: dovecot-2.2: master: host.domain lookups weren't cached, they we... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/69179ca6007d changeset: 16839:69179ca6007d user: Timo Sirainen date: Thu Oct 10 20:50:10 2013 +0300 description: master: host.domain lookups weren't cached, they were done at the worst possible time. This fixes "net_connect_unix(imap) failed: Resource temporarily unavailable" and various other issues where processes weren't being created fast enough if the host.domain lookup was slow (e.g. done via external DNS). diffstat: src/master/service-process.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (49 lines): diff -r dc46ae14008c -r 69179ca6007d src/master/service-process.c --- a/src/master/service-process.c Tue Oct 08 16:48:04 2013 +0300 +++ b/src/master/service-process.c Thu Oct 10 20:50:10 2013 +0300 @@ -237,7 +237,8 @@ } static void -service_process_setup_environment(struct service *service, unsigned int uid) +service_process_setup_environment(struct service *service, unsigned int uid, + const char *hostdomain) { master_service_env_clean(); @@ -257,7 +258,7 @@ } env_put(t_strdup_printf(MASTER_UID_ENV"=%u", uid)); env_put(t_strdup_printf(MY_HOSTNAME_ENV"=%s", my_hostname)); - env_put(t_strdup_printf(MY_HOSTDOMAIN_ENV"=%s", my_hostdomain())); + env_put(t_strdup_printf(MY_HOSTDOMAIN_ENV"=%s", hostdomain)); if (!service->set->master_set->version_ignore) env_put(MASTER_DOVECOT_VERSION_ENV"="PACKAGE_VERSION); @@ -291,6 +292,7 @@ static unsigned int uid_counter = 0; struct service_process *process; unsigned int uid = ++uid_counter; + const char *hostdomain; pid_t pid; bool process_forked; @@ -305,6 +307,9 @@ new processes now */ return NULL; } + /* look this up before fork()ing so that it gets cached for all the + future lookups. */ + hostdomain = my_hostdomain(); if (service->type == SERVICE_TYPE_ANVIL && service_anvil_global->pid != 0) { @@ -323,7 +328,7 @@ } if (pid == 0) { /* child */ - service_process_setup_environment(service, uid); + service_process_setup_environment(service, uid, hostdomain); service_reopen_inet_listeners(service); service_dup_fds(service); drop_privileges(service); From pigeonhole at rename-it.nl Fri Oct 11 08:57:29 2013 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 11 Oct 2013 07:57:29 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: vacation extension: Fixed int... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ab3638d7aa4a changeset: 1812:ab3638d7aa4a user: Stephan Bosch date: Fri Oct 11 07:57:21 2013 +0200 description: lib-sieve: vacation extension: Fixed interaction of sieve_vacation_dont_check_recipient with sieve_vacation_send_from_recipient setting. If sieve_vacation_dont_check_recipient was active, sieve_vacation_send_from_recipient had no effect. diffstat: src/lib-sieve/plugins/vacation/cmd-vacation.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 689dbc10a7fc -r ab3638d7aa4a src/lib-sieve/plugins/vacation/cmd-vacation.c --- a/src/lib-sieve/plugins/vacation/cmd-vacation.c Sun Oct 06 22:14:01 2013 +0200 +++ b/src/lib-sieve/plugins/vacation/cmd-vacation.c Fri Oct 11 07:57:21 2013 +0200 @@ -1199,6 +1199,7 @@ if ( config->dont_check_recipient ) { /* Send reply from envelope recipient address */ reply_from = recipient; + smtp_from = recipient; } else { const char *original_recipient = ""; From dovecot at dovecot.org Fri Oct 11 12:23:03 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 12:23:03 +0300 Subject: dovecot-2.2: lib-storage: Header+body searches might not have se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/78081ecd7eb3 changeset: 16840:78081ecd7eb3 user: Timo Sirainen date: Fri Oct 11 12:22:56 2013 +0300 description: lib-storage: Header+body searches might not have searched the body in some cases. Only the header was requested for the search stream, which with imapc and pop3c could have caused the body not to be in the stream, although it usually was because of the "fetch body" hint. diffstat: src/lib-storage/index/index-search.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 69179ca6007d -r 78081ecd7eb3 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Thu Oct 10 20:50:10 2013 +0300 +++ b/src/lib-storage/index/index-search.c Fri Oct 11 12:22:56 2013 +0300 @@ -679,7 +679,10 @@ input = NULL; } else if (have_headers) { /* we need to read the entire header */ - if (mail_get_hdr_stream(ctx->cur_mail, NULL, &input) < 0) + ret = have_body ? + mail_get_stream(ctx->cur_mail, NULL, NULL, &input) : + mail_get_hdr_stream(ctx->cur_mail, NULL, &input); + if (ret < 0) failed = TRUE; else { hdr_ctx.parse_headers = From dovecot at dovecot.org Fri Oct 11 12:32:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 12:32:08 +0300 Subject: dovecot-2.2: lib-storage: If search finds a match to a cached he... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4b0a736bc40c changeset: 16841:4b0a736bc40c user: Timo Sirainen date: Fri Oct 11 12:31:48 2013 +0300 description: lib-storage: If search finds a match to a cached header, don't open the body stream. diffstat: src/lib-storage/index/index-search.c | 2 +- src/lib-storage/mail-search.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 78081ecd7eb3 -r 4b0a736bc40c src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Fri Oct 11 12:22:56 2013 +0300 +++ b/src/lib-storage/index/index-search.c Fri Oct 11 12:31:48 2013 +0300 @@ -665,7 +665,7 @@ mailbox_header_lookup_init(ctx->box, headers); if (headers != NULL && (!have_body || - ctx->cur_mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER)) { + ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)) { /* try to look up the specified headers from cache */ i_assert(*headers != NULL); diff -r 78081ecd7eb3 -r 4b0a736bc40c src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Fri Oct 11 12:22:56 2013 +0300 +++ b/src/lib-storage/mail-search.c Fri Oct 11 12:31:48 2013 +0300 @@ -520,7 +520,7 @@ *have_headers = have_text || headers->used != 0; - if (headers->used == 0 || have_text) + if (headers->used == 0) return NULL; buffer_append(headers, &null, sizeof(const char *)); From dovecot at dovecot.org Fri Oct 11 18:55:10 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 18:55:10 +0300 Subject: dovecot-2.2: LAYOUT=index: Fixed race condition during mailbox c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ab341f7509b5 changeset: 16842:ab341f7509b5 user: Timo Sirainen date: Fri Oct 11 18:54:36 2013 +0300 description: LAYOUT=index: Fixed race condition during mailbox creation. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 33 ++++++++++++---------- src/lib-storage/list/mailbox-list-index-sync.c | 4 ++ src/lib-storage/list/mailbox-list-index.h | 1 + 3 files changed, 23 insertions(+), 15 deletions(-) diffs (141 lines): diff -r 4b0a736bc40c -r ab341f7509b5 src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Fri Oct 11 12:31:48 2013 +0300 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Fri Oct 11 18:54:36 2013 +0300 @@ -136,7 +136,10 @@ } return -1; } - view = mail_index_view_open(ilist->index); + /* we could get here during sync from + index_list_mailbox_create_selectable() */ + view = ilist->sync_ctx == NULL ? mail_index_view_open(ilist->index) : + ilist->sync_ctx->view; if (!mail_index_lookup_seq(view, node->uid, &seq)) i_panic("mailbox list index: lost uid=%u", node->uid); if (!mailbox_list_index_status(_list, view, seq, 0, @@ -149,7 +152,8 @@ *path_r = index_get_guid_path(_list, root_dir, mailbox_guid); ret = 1; } - mail_index_view_close(&view); + if (ilist->sync_ctx == NULL) + mail_index_view_close(&view); return ret; } @@ -234,20 +238,23 @@ } static int -index_list_mailbox_create_selectable(struct index_mailbox_list *list, - const char *name, guid_128_t mailbox_guid) +index_list_mailbox_create_selectable(struct mailbox *box, + const struct mailbox_update *update) { + struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box); + struct index_mailbox_list *list = + (struct index_mailbox_list *)box->list; struct mailbox_list_index_sync_context *sync_ctx; struct mailbox_list_index_record rec; struct mailbox_list_index_node *node; const void *data; - bool expunged, created; + bool expunged, created, success; uint32_t seq; if (mailbox_list_index_sync_begin(&list->list, &sync_ctx) < 0) return -1; - seq = mailbox_list_index_sync_name(sync_ctx, name, &node, &created); + seq = mailbox_list_index_sync_name(sync_ctx, box->name, &node, &created); if (!created && (node->flags & (MAILBOX_LIST_INDEX_FLAG_NONEXISTENT | MAILBOX_LIST_INDEX_FLAG_NOSELECT)) == 0) { @@ -266,11 +273,12 @@ node->flags = 0; mail_index_update_flags(sync_ctx->trans, seq, MODIFY_REPLACE, 0); - memcpy(rec.guid, mailbox_guid, sizeof(rec.guid)); + memcpy(rec.guid, update->mailbox_guid, sizeof(rec.guid)); mail_index_update_ext(sync_ctx->trans, seq, sync_ctx->ilist->ext_id, &rec, NULL); - if (mailbox_list_index_sync_end(&sync_ctx, TRUE) < 0) + success = ibox->module_ctx.super.create_box(box, update, FALSE) == 0; + if (mailbox_list_index_sync_end(&sync_ctx, success) < 0) return -1; return 1; } @@ -279,7 +287,6 @@ index_list_mailbox_create(struct mailbox *box, const struct mailbox_update *update, bool directory) { - struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box); struct index_mailbox_list *list = (struct index_mailbox_list *)box->list; struct mailbox_update new_update; @@ -306,14 +313,11 @@ new_update = *update; if (guid_128_is_empty(new_update.mailbox_guid)) guid_128_generate(new_update.mailbox_guid); - ret = index_list_mailbox_create_selectable(list, box->name, - new_update.mailbox_guid); + ret = index_list_mailbox_create_selectable(box, &new_update); if (ret < 0) { mail_storage_copy_list_error(box->storage, box->list); return -1; } - /* the storage backend needs to use the same GUID */ - update = &new_update; } else { ret = 0; } @@ -323,8 +327,7 @@ "Mailbox already exists"); return -1; } - return directory ? 0 : - ibox->module_ctx.super.create_box(box, update, directory); + return 0; } static int diff -r 4b0a736bc40c -r ab341f7509b5 src/lib-storage/list/mailbox-list-index-sync.c --- a/src/lib-storage/list/mailbox-list-index-sync.c Fri Oct 11 12:31:48 2013 +0300 +++ b/src/lib-storage/list/mailbox-list-index-sync.c Fri Oct 11 18:54:36 2013 +0300 @@ -236,6 +236,8 @@ struct mail_index_transaction *trans; const struct mail_index_header *hdr; + i_assert(!ilist->syncing); + if (mail_index_sync_begin(ilist->index, &index_sync_ctx, &view, &trans, MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES) < 0) { mailbox_list_index_set_index_error(list); @@ -269,6 +271,7 @@ &uid_validity, sizeof(uid_validity), TRUE); } sync_ctx->view = mail_index_transaction_open_updated_view(trans); + ilist->sync_ctx = sync_ctx; ilist->syncing = TRUE; *sync_ctx_r = sync_ctx; @@ -370,6 +373,7 @@ ret = -1; } sync_ctx->ilist->syncing = FALSE; + sync_ctx->ilist->sync_ctx = NULL; i_free(sync_ctx); return ret; } diff -r 4b0a736bc40c -r ab341f7509b5 src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Fri Oct 11 12:31:48 2013 +0300 +++ b/src/lib-storage/list/mailbox-list-index.h Fri Oct 11 18:54:36 2013 +0300 @@ -92,6 +92,7 @@ HASH_TABLE(void *, char *) mailbox_names; uint32_t highest_name_id; + struct mailbox_list_index_sync_context *sync_ctx; uint32_t sync_log_file_seq; uoff_t sync_log_file_offset; uint32_t sync_stamp; From dovecot at dovecot.org Fri Oct 11 19:18:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 19:18:48 +0300 Subject: dovecot-2.2: liblib: If parent ostream closes itself on error, c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff2452560ef0 changeset: 16843:ff2452560ef0 user: Timo Sirainen date: Fri Oct 11 19:17:10 2013 +0300 description: liblib: If parent ostream closes itself on error, close our ostream as well. This avoids a situation where ostream is basically unusable with last_failed_errno set, but it's not marked as closed. The current code often checks ostream->closed but doesn't check last_failed_errno. ostream-file also autocloses the stream, but filter ostreams without this patch don't autoclose, so this caused problems with e.g. IMAP COMPRESSION extension where the zlib-ostream didn't get marked as closed, although the problem was only logging "BUG: Unknown internal error" instead of "Disconnected" as the client's disconnect reason. diffstat: src/lib/ostream.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (25 lines): diff -r ab341f7509b5 -r ff2452560ef0 src/lib/ostream.c --- a/src/lib/ostream.c Fri Oct 11 18:54:36 2013 +0300 +++ b/src/lib/ostream.c Fri Oct 11 19:17:10 2013 +0300 @@ -45,8 +45,10 @@ static void o_stream_close_full(struct ostream *stream, bool close_parents) { - io_stream_close(&stream->real_stream->iostream, close_parents); - stream->closed = TRUE; + if (!stream->closed) { + io_stream_close(&stream->real_stream->iostream, close_parents); + stream->closed = TRUE; + } if (stream->stream_errno == 0) stream->stream_errno = EPIPE; @@ -428,6 +430,8 @@ dest->stream_errno = src->stream_errno; dest->last_failed_errno = src->last_failed_errno; dest->overflow = src->overflow; + if (src->closed) + o_stream_close(dest); } static int o_stream_default_flush(struct ostream_private *_stream) From dovecot at dovecot.org Fri Oct 11 19:27:54 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 19:27:54 +0300 Subject: dovecot-2.2: imap: Improved "EOF while appending" disconnect log... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2e23267dff66 changeset: 16844:2e23267dff66 user: Timo Sirainen date: Fri Oct 11 19:27:39 2013 +0300 description: imap: Improved "EOF while appending" disconnect log message. diffstat: src/imap/cmd-append.c | 14 ++++++++------ 1 files changed, 8 insertions(+), 6 deletions(-) diffs (56 lines): diff -r ff2452560ef0 -r 2e23267dff66 src/imap/cmd-append.c --- a/src/imap/cmd-append.c Fri Oct 11 19:17:10 2013 +0300 +++ b/src/imap/cmd-append.c Fri Oct 11 19:27:39 2013 +0300 @@ -52,7 +52,8 @@ static bool cmd_append_continue_message(struct client_command_context *cmd); static bool cmd_append_parse_new_msg(struct client_command_context *cmd); -static const char *get_disconnect_reason(struct cmd_append_context *ctx) +static const char * +get_disconnect_reason(struct cmd_append_context *ctx, uoff_t lit_offset) { string_t *str = t_str_new(128); unsigned int secs = ioloop_time - ctx->started; @@ -61,7 +62,7 @@ ctx->count, secs); if (ctx->litinput != NULL) { str_printfa(str, ", %"PRIuUOFF_T"/%"PRIuUOFF_T" bytes", - ctx->litinput->v_offset, ctx->literal_size); + lit_offset, ctx->literal_size); } str_append_c(str, ')'); return str_c(str); @@ -82,7 +83,7 @@ switch (i_stream_read(client->input)) { case -1: /* disconnected */ - reason = get_disconnect_reason(ctx); + reason = get_disconnect_reason(ctx, ctx->litinput->v_offset); cmd_append_finish(cmd->context); /* Reset command so that client_destroy() doesn't try to call cmd_append_continue_message() anymore. */ @@ -825,7 +826,7 @@ } if (ctx->litinput->eof || client->input->closed) { - bool all_written = ctx->litinput->v_offset == ctx->literal_size; + uoff_t lit_offset = ctx->litinput->v_offset; /* finished - do one more read, to make sure istream-chain unreferences its stream, which is needed for litinput's @@ -843,12 +844,13 @@ /* failed above */ client_send_storage_error(cmd, ctx->storage); ctx->failed = TRUE; - } else if (!all_written) { + } else if (lit_offset != ctx->literal_size) { /* client disconnected before it finished sending the whole message. */ ctx->failed = TRUE; mailbox_save_cancel(&ctx->save_ctx); - client_disconnect(client, "EOF while appending"); + client_disconnect(client, + get_disconnect_reason(ctx, lit_offset)); } else if (ctx->catenate) { /* CATENATE isn't finished yet */ } else if (mailbox_save_finish(&ctx->save_ctx) < 0) { From dovecot at dovecot.org Fri Oct 11 20:03:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 20:03:21 +0300 Subject: dovecot-2.2: lib-lda: lmtp_client_connect_tcp() shouldn't immedi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/086c75564ba0 changeset: 16845:086c75564ba0 user: Timo Sirainen date: Fri Oct 11 20:03:06 2013 +0300 description: lib-lda: lmtp_client_connect_tcp() shouldn't immediately call callbacks if dns lookup fails. diffstat: src/lib-lda/lmtp-client.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (36 lines): diff -r 2e23267dff66 -r 086c75564ba0 src/lib-lda/lmtp-client.c --- a/src/lib-lda/lmtp-client.c Fri Oct 11 19:27:39 2013 +0300 +++ b/src/lib-lda/lmtp-client.c Fri Oct 11 20:03:06 2013 +0300 @@ -69,6 +69,7 @@ struct istream *data_input; unsigned char output_last; + unsigned int running:1; unsigned int xclient_sent:1; unsigned int rcpt_to_successes:1; unsigned int output_finished:1; @@ -617,11 +618,13 @@ if (result->ret != 0) { i_error("lmtp client: DNS lookup of %s failed: %s", client->host, result->error); - lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE - " (DNS lookup)"); + if (client->running) { + lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE + " (DNS lookup)"); + } } else { client->ip = result->ips[0]; - if (lmtp_client_connect(client) < 0) { + if (lmtp_client_connect(client) < 0 && client->running) { lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE " (connect)"); } @@ -668,6 +671,7 @@ lmtp_client_dns_done, client, &client->dns_lookup) < 0) return -1; + client->running = TRUE; return 0; } From dovecot at dovecot.org Fri Oct 11 21:37:13 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 11 Oct 2013 21:37:13 +0300 Subject: dovecot-2.2: lib-dns: Close dns-client fd before calling callback. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a677a70e420c changeset: 16846:a677a70e420c user: Timo Sirainen date: Fri Oct 11 21:36:21 2013 +0300 description: lib-dns: Close dns-client fd before calling callback. Just in case the callback is slow and keeps the fd unnecessarily long open. diffstat: src/lib-dns/dns-lookup.c | 25 ++++++++++++++++++------- 1 files changed, 18 insertions(+), 7 deletions(-) diffs (56 lines): diff -r 086c75564ba0 -r a677a70e420c src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Fri Oct 11 20:03:06 2013 +0300 +++ b/src/lib-dns/dns-lookup.c Fri Oct 11 21:36:21 2013 +0300 @@ -36,6 +36,21 @@ static void dns_lookup_free(struct dns_lookup **_lookup); +static void dns_lookup_close(struct dns_lookup *lookup) +{ + if (lookup->to != NULL) + timeout_remove(&lookup->to); + if (lookup->io != NULL) + io_remove(&lookup->io); + if (lookup->input != NULL) + i_stream_destroy(&lookup->input); + if (lookup->fd != -1) { + if (close(lookup->fd) < 0) + i_error("close(%s) failed: %m", lookup->path); + lookup->fd = -1; + } +} + static int dns_lookup_input_line(struct dns_lookup *lookup, const char *line) { struct dns_lookup_result *result = &lookup->result; @@ -122,6 +137,7 @@ } if (ret != 0) { dns_lookup_save_msecs(lookup); + dns_lookup_close(lookup); lookup->callback(result, lookup->context); dns_lookup_free(&lookup); } @@ -131,6 +147,7 @@ { lookup->result.error = "DNS lookup timed out"; + dns_lookup_close(lookup); lookup->callback(&lookup->result, lookup->context); dns_lookup_free(&lookup); } @@ -209,13 +226,7 @@ *_lookup = NULL; - if (lookup->to != NULL) - timeout_remove(&lookup->to); - io_remove(&lookup->io); - i_stream_destroy(&lookup->input); - if (close(lookup->fd) < 0) - i_error("close(%s) failed: %m", lookup->path); - + dns_lookup_close(lookup); i_free(lookup->name); i_free(lookup->ips); i_free(lookup->path); From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: Made dummy response for retu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa39d877d2c5 changeset: 16848:aa39d877d2c5 user: Stephan Bosch date: Sat Oct 12 10:53:16 2013 +0300 description: lib-http: http-client: Made dummy response for returning internal errors more complete by filling in the version number and a proper date. diffstat: src/lib-http/http-response.c | 13 +++++++++++++ src/lib-http/http-response.h | 9 ++------- 2 files changed, 15 insertions(+), 7 deletions(-) diffs (46 lines): diff -r 74bc909ba6a0 -r aa39d877d2c5 src/lib-http/http-response.c --- a/src/lib-http/http-response.c Sat Oct 12 10:52:35 2013 +0300 +++ b/src/lib-http/http-response.c Sat Oct 12 10:53:16 2013 +0300 @@ -1,9 +1,22 @@ #include "lib.h" #include "array.h" +#include "ioloop.h" #include "istream.h" #include "http-response.h" +void +http_response_init(struct http_response *resp, + unsigned int status, const char *reason) +{ + memset(resp, 0, sizeof(*resp)); + resp->version_major = 1; + resp->version_minor = 1; + resp->date = ioloop_time; + resp->status = status; + resp->reason = reason; +} + bool http_response_has_connection_option(const struct http_response *resp, const char *option) { diff -r 74bc909ba6a0 -r aa39d877d2c5 src/lib-http/http-response.h --- a/src/lib-http/http-response.h Sat Oct 12 10:52:35 2013 +0300 +++ b/src/lib-http/http-response.h Sat Oct 12 10:53:16 2013 +0300 @@ -28,14 +28,9 @@ unsigned int connection_close:1; }; -static inline void +void http_response_init(struct http_response *resp, - unsigned int status, const char *reason) -{ - memset(resp, 0, sizeof(*resp)); - resp->status = status; - resp->reason = reason; -} + unsigned int status, const char *reason); static inline const struct http_header_field * http_response_header_find(const struct http_response *resp, const char *name) From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: When client request is retri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ad46c778981d changeset: 16849:ad46c778981d user: Stephan Bosch date: Sat Oct 12 10:54:10 2013 +0300 description: lib-http: http-client: When client request is retried based on a response received from server, return full response to caller if retry is not possible. Before, it would construct a dummy-response that basically retains only the status and reason elements. This is currently only relevant for the way a 417 Expectation Failed response from the server is handled. diffstat: src/lib-http/http-client-connection.c | 2 +- src/lib-http/http-client-private.h | 2 ++ src/lib-http/http-client-request.c | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletions(-) diffs (44 lines): diff -r aa39d877d2c5 -r ad46c778981d src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Oct 12 10:53:16 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sat Oct 12 10:54:10 2013 +0300 @@ -593,7 +593,7 @@ req->payload_sync = FALSE; conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; - http_client_request_retry(req, response.status, response.reason); + http_client_request_retry_response(req, &response); } else if (response.status / 100 == 3 && response.status != 304 && response.location != NULL) { diff -r aa39d877d2c5 -r ad46c778981d src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Oct 12 10:53:16 2013 +0300 +++ b/src/lib-http/http-client-private.h Sat Oct 12 10:54:10 2013 +0300 @@ -240,6 +240,8 @@ void http_client_request_resubmit(struct http_client_request *req); void http_client_request_retry(struct http_client_request *req, unsigned int status, const char *error); +void http_client_request_retry_response(struct http_client_request *req, + struct http_response *response); void http_client_request_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_redirect(struct http_client_request *req, diff -r aa39d877d2c5 -r ad46c778981d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Oct 12 10:53:16 2013 +0300 +++ b/src/lib-http/http-client-request.c Sat Oct 12 10:54:10 2013 +0300 @@ -773,6 +773,16 @@ http_client_request_error(req, status, error); } +void http_client_request_retry_response(struct http_client_request *req, + struct http_response *response) +{ + if (!http_client_request_try_retry(req)) { + i_assert(req->submitted || req->state >= HTTP_REQUEST_STATE_FINISHED); + (void)http_client_request_callback(req, response); + http_client_request_unref(&req); + } +} + bool http_client_request_try_retry(struct http_client_request *req) { /* limit the number of attempts for each request */ From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: Added more tests for the http_request_par... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/74bc909ba6a0 changeset: 16847:74bc909ba6a0 user: Stephan Bosch date: Sat Oct 12 10:52:35 2013 +0300 description: lib-http: Added more tests for the http_request_parser. diffstat: src/lib-http/test-http-request-parser.c | 249 ++++++++++++++++++++++++++----- 1 files changed, 209 insertions(+), 40 deletions(-) diffs (truncated from 365 to 300 lines): diff -r a677a70e420c -r 74bc909ba6a0 src/lib-http/test-http-request-parser.c --- a/src/lib-http/test-http-request-parser.c Fri Oct 11 21:36:21 2013 +0300 +++ b/src/lib-http/test-http-request-parser.c Sat Oct 12 10:52:35 2013 +0300 @@ -12,20 +12,27 @@ #include -struct http_request_parse_test { +/* + * Test: valid requests + */ + +struct http_request_valid_parse_test { const char *request; const char *method; const char *target_raw; - struct http_request_target target; + struct { + enum http_request_target_format format; + struct http_url url; + } target; unsigned char version_major; unsigned char version_minor; uoff_t content_length; const char *payload; + bool connection_close; + bool expect_100_continue; }; -/* Valid header tests */ - -static const struct http_request_parse_test +static const struct http_request_valid_parse_test valid_request_parse_tests[] = { { .request = "GET / HTTP/1.1\r\n" @@ -34,17 +41,20 @@ .method = "GET", .target_raw = "/", .target = { - .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN + .format = HTTP_REQUEST_TARGET_FORMAT_ORIGIN, + .url = { .host_name = "example.com" } }, .version_major = 1, .version_minor = 1, },{ .request = "OPTIONS * HTTP/1.0\r\n" "Host: example.com\r\n" + "Connection: Keep-Alive\r\n" "\r\n", .method = "OPTIONS", .target_raw = "*", .target = { - .format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK + .format = HTTP_REQUEST_TARGET_FORMAT_ASTERISK, + .url = { .host_name = "example.com" } }, .version_major = 1, .version_minor = 0, },{ .request = @@ -54,7 +64,8 @@ .method = "CONNECT", .target_raw = "example.com:443", .target = { - .format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY + .format = HTTP_REQUEST_TARGET_FORMAT_AUTHORITY, + .url = { .host_name = "example.com", .have_port = TRUE, .port = 443 } }, .version_major = 1, .version_minor = 2, },{ .request = @@ -64,7 +75,12 @@ .method = "GET", .target_raw = "https://www.example.com:443", .target = { - .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { + .host_name = "www.example.com", + .have_port = TRUE, .port = 443, + .have_ssl = TRUE + } }, .version_major = 1, .version_minor = 1, },{ .request = @@ -77,9 +93,48 @@ .target_raw = "http://api.example.com:8080/commit?user=dirk", .target = { .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { .host_name = "api.example.com", .have_port = TRUE, .port = 8080 } }, .version_major = 1, .version_minor = 1, .payload = "Content!\r\n" + },{ .request = + "GET http://www.example.com/index.php?seq=1 HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Connection: close\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.php?seq=1", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { .host_name = "www.example.com" } + }, + .version_major = 1, .version_minor = 1, + .connection_close = TRUE + },{ .request = + "GET http://www.example.com/index.html HTTP/1.0\r\n" + "Host: www.example.com\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.html", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { .host_name = "www.example.com" } + }, + .version_major = 1, .version_minor = 0, + .connection_close = TRUE + },{ .request = + "GET http://www.example.com/index.html HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Expect: 100-continue\r\n" + "\r\n", + .method = "GET", + .target_raw = "http://www.example.com/index.html", + .target = { + .format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE, + .url = { .host_name = "www.example.com" } + }, + .version_major = 1, .version_minor = 1, + .expect_100_continue = TRUE } }; @@ -110,7 +165,7 @@ for (i = 0; i < valid_request_parse_test_count; i++) T_BEGIN { struct istream *input; struct ostream *output; - const struct http_request_parse_test *test; + const struct http_request_valid_parse_test *test; struct http_request_parser *parser; struct http_request request; enum http_request_parse_error error_code; @@ -153,26 +208,65 @@ if (ret == 0) { /* verify last request only */ if (request.method == NULL || test->method == NULL) { - test_out(t_strdup_printf("request->method = %s", test->method), + test_out(t_strdup_printf("request->method = %s", request.method), request.method == test->method); } else { - test_out(t_strdup_printf("request->method = %s", test->method), + test_out(t_strdup_printf("request->method = %s", request.method), strcmp(request.method, test->method) == 0); } + if (request.target_raw == NULL || test->target_raw == NULL) { - test_out(t_strdup_printf("request->target = %s", test->target_raw), + test_out(t_strdup_printf + ("request->target_raw = %s", request.target_raw), request.target_raw == test->target_raw); } else { - test_out(t_strdup_printf("request->target = %s", test->target_raw), + test_out(t_strdup_printf + ("request->target_raw = %s", request.target_raw), strcmp(request.target_raw, test->target_raw) == 0); } + if (request.target.url == NULL) { + test_out("request->target.url = (null)", + test->target.url.host_name == NULL && !test->target.url.have_port); + } else { + if (request.target.url->host_name == NULL || + test->target.url.host_name == NULL) { + test_out(t_strdup_printf("request->target.url->host_name = %s", + request.target.url->host_name), + request.target.url->host_name == test->target.url.host_name); + } else { + test_out(t_strdup_printf("request->target.url->host_name = %s", + request.target.url->host_name), + strcmp(request.target.url->host_name, + test->target.url.host_name) == 0); + } + if (!request.target.url->have_port) { + test_out("request->target.url->port = (unspecified)", + request.target.url->have_port == test->target.url.have_port); + } else { + test_out(t_strdup_printf + ("request->target.url->port = %u", request.target.url->port), + request.target.url->have_port == test->target.url.have_port && + request.target.url->port == test->target.url.port); + } + test_out(t_strdup_printf("request->target.url->have_ssl = %s", + (request.target.url->have_ssl ? "yes" : "no")), + request.target.url->have_ssl == test->target.url.have_ssl); + } test_out(t_strdup_printf("request->target_format = %s", - _request_target_format(test->target.format)), + _request_target_format(request.target.format)), request.target.format == test->target.format); - test_out(t_strdup_printf("request->version = %d.%d", - test->version_major, test->version_minor), + + test_out(t_strdup_printf("request->version = %u.%u", + request.version_major, request.version_minor), request.version_major == test->version_major && request.version_minor == test->version_minor); + test_out(t_strdup_printf("request->connection_close = %s", + (request.connection_close ? "yes" : "no")), + request.connection_close == test->connection_close); + test_out(t_strdup_printf("request->expect_100_continue = %s", + (request.expect_100_continue ? "yes" : "no")), + request.expect_100_continue == test->expect_100_continue); + if (payload == NULL || test->payload == NULL) { test_out(t_strdup_printf("request->payload = %s", str_sanitize(payload, 80)), @@ -190,25 +284,71 @@ buffer_free(&payload_buffer); } -static const char *invalid_request_parse_tests[] = { - "GET: / HTTP/1.1\r\n" - "Host: example.com\r\n" - "\r\n", - "GET % HTTP/1.1\r\n" - "Host: example.com\r\n" - "\r\n", - "GET /frop\" HTTP/1.1\r\n" - "Host: example.com\r\n" - "\r\n", - "GET / HTCPCP/1.0\r\n" - "Host: example.com\r\n" - "\r\n", - "GET / HTTP/1.0.1\r\n" - "Host: example.com\r\n" - "\r\n", - "GET / HTTP/1.1\r\n" - "Host: \"example.com\r\n" - "\r\n", +/* + * Test: invalid requests + */ + +struct http_request_invalid_parse_test { + const char *request; + enum http_request_parse_error error_code; +}; + +static struct http_request_invalid_parse_test +invalid_request_parse_tests[] = { + { .request = + "GET: / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + },{ .request = + "GET % HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + },{ .request = + "GET /frop\" HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + },{ .request = + "GET / HTCPCP/1.0\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + },{ .request = + "GET / HTTP/1.0.1\r\n" + "Host: example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + },{ .request = + "GET / HTTP/1.1\r\n" + "Host: \"example.com\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + },{ .request = + "GET / HTTP/1.1\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BAD_REQUEST + },{ .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Transfer-Encoding: gzip\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_BROKEN_REQUEST + },{ .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Expect: payment\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_EXPECTATION_FAILED + },{ .request = + "GET / HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Transfer-Encoding: cuneiform, chunked\r\n" + "\r\n", + .error_code = HTTP_REQUEST_PARSE_ERROR_NOT_IMPLEMENTED From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: Made requests release payloa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2d44e9025af3 changeset: 16850:2d44e9025af3 user: Stephan Bosch date: Sat Oct 12 10:54:50 2013 +0300 description: lib-http: http-client: Made requests release payload input stream as early as possible. This prevents deadlock conditions when used for the HTTP proxy. diffstat: src/lib-http/http-client-request.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (25 lines): diff -r ad46c778981d -r 2d44e9025af3 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Oct 12 10:54:10 2013 +0300 +++ b/src/lib-http/http-client-request.c Sat Oct 12 10:54:50 2013 +0300 @@ -543,6 +543,10 @@ req->callback = callback; http_client_request_resubmit(req); return FALSE; + } else { + /* release payload early (prevents server/client deadlock in proxy) */ + if (req->payload_input != NULL) + i_stream_unref(&req->payload_input); } } return TRUE; @@ -563,6 +567,10 @@ http_response_init(&response, status, error); (void)callback(&response, req->context); + + /* release payload early (prevents server/client in proxy) */ + if (req->payload_input != NULL) + i_stream_unref(&req->payload_input); } } From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: Added support for disabling automatic red... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4d54b6f3d10 changeset: 16851:b4d54b6f3d10 user: Stephan Bosch date: Sat Oct 12 10:55:38 2013 +0300 description: lib-http: Added support for disabling automatic redirects. diffstat: src/lib-http/http-client-connection.c | 5 +++-- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diffs (39 lines): diff -r 2d44e9025af3 -r b4d54b6f3d10 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Oct 12 10:54:50 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sat Oct 12 10:55:38 2013 +0300 @@ -594,8 +594,9 @@ conn->output_locked = FALSE; conn->peer->no_payload_sync = TRUE; http_client_request_retry_response(req, &response); - - } else if (response.status / 100 == 3 && response.status != 304 && + + } else if (!req->client->set.no_auto_redirect && + response.status / 100 == 3 && response.status != 304 && response.location != NULL) { /* redirect */ http_client_request_redirect(req, response.status, response.location); diff -r 2d44e9025af3 -r b4d54b6f3d10 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Sat Oct 12 10:54:50 2013 +0300 +++ b/src/lib-http/http-client.c Sat Oct 12 10:55:38 2013 +0300 @@ -93,6 +93,7 @@ client->set.max_pipelined_requests = (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); client->set.max_attempts = set->max_attempts; + client->set.no_auto_redirect = set->no_auto_redirect; client->set.max_redirects = set->max_redirects; client->set.response_hdr_limits = set->response_hdr_limits; client->set.request_timeout_msecs = set->request_timeout_msecs; diff -r 2d44e9025af3 -r b4d54b6f3d10 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Sat Oct 12 10:54:50 2013 +0300 +++ b/src/lib-http/http-client.h Sat Oct 12 10:55:38 2013 +0300 @@ -54,6 +54,9 @@ /* maximum number of pipelined requests per connection (default = 1) */ unsigned int max_pipelined_requests; + /* don't automatically act upon redirect responses */ + bool no_auto_redirect; + /* maximum number of redirects for a request (default = 0; redirects refused) */ From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: Added request error code for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a8fe88d2286d changeset: 16852:a8fe88d2286d user: Stephan Bosch date: Sat Oct 12 10:57:05 2013 +0300 description: lib-http: http-client: Added request error code for broken payload input stream. This error is triggered when reading from the provided payload input stream fails while sending the request. Previously this would yield the same error code as for a failure to write to the connection output. diffstat: src/lib-http/http-client-connection.c | 6 ++++-- src/lib-http/http-client-private.h | 4 +++- src/lib-http/http-client-request.c | 16 ++++++++++++++-- src/lib-http/http-client.h | 1 + 4 files changed, 22 insertions(+), 5 deletions(-) diffs (112 lines): diff -r b4d54b6f3d10 -r a8fe88d2286d src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Oct 12 10:55:38 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sat Oct 12 10:57:05 2013 +0300 @@ -74,7 +74,8 @@ struct http_client_request **req; array_foreach_modifiable(&conn->request_wait_list, req) { - http_client_request_retry(*req, status, error); + if ((*req)->state < HTTP_REQUEST_STATE_FINISHED) + http_client_request_retry(*req, status, error); http_client_request_unref(req); } array_clear(&conn->request_wait_list); @@ -93,7 +94,8 @@ "Server explicitly closed connection"); array_foreach_modifiable(&conn->request_wait_list, req) { - http_client_request_resubmit(*req); + if ((*req)->state < HTTP_REQUEST_STATE_FINISHED) + http_client_request_resubmit(*req); http_client_request_unref(req); } array_clear(&conn->request_wait_list); diff -r b4d54b6f3d10 -r a8fe88d2286d src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Oct 12 10:55:38 2013 +0300 +++ b/src/lib-http/http-client-private.h Sat Oct 12 10:57:05 2013 +0300 @@ -232,7 +232,7 @@ void http_client_request_ref(struct http_client_request *req); void http_client_request_unref(struct http_client_request **_req); int http_client_request_send(struct http_client_request *req, - const char **error_r); + const char **error_r); int http_client_request_send_more(struct http_client_request *req, const char **error_r); bool http_client_request_callback(struct http_client_request *req, @@ -242,6 +242,8 @@ unsigned int status, const char *error); void http_client_request_retry_response(struct http_client_request *req, struct http_response *response); +void http_client_request_send_error(struct http_client_request *req, + unsigned int status, const char *error); void http_client_request_error(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_redirect(struct http_client_request *req, diff -r b4d54b6f3d10 -r a8fe88d2286d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Oct 12 10:55:38 2013 +0300 +++ b/src/lib-http/http-client-request.c Sat Oct 12 10:57:05 2013 +0300 @@ -377,11 +377,20 @@ o_stream_set_max_buffer_size(output, (size_t)-1); if (req->payload_input->stream_errno != 0) { + /* the payload stream assigned to this request is broken, + fail this the request immediately */ + http_client_request_send_error(req, + HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, + "Broken payload stream"); + + /* we're in the middle of sending a request, so the connection + will also have to be aborted */ errno = req->payload_input->stream_errno; *error_r = t_strdup_printf("read(%s) failed: %m", i_stream_get_name(req->payload_input)); ret = -1; } else if (output->stream_errno != 0) { + /* failed to send request */ errno = output->stream_errno; *error_r = t_strdup_printf("write(%s) failed: %m", o_stream_get_name(output)); @@ -393,6 +402,7 @@ if (ret < 0 || i_stream_is_eof(req->payload_input)) { if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { + *error_r = "stream input size changed [BUG]"; i_error("stream input size changed"); //FIXME return -1; } @@ -552,12 +562,14 @@ return TRUE; } -static void +void http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error) { http_client_request_callback_t *callback; + if (req->state >= HTTP_REQUEST_STATE_FINISHED) + return; req->state = HTTP_REQUEST_STATE_ABORTED; callback = req->callback; @@ -603,7 +615,7 @@ void http_client_request_error(struct http_client_request *req, unsigned int status, const char *error) { - if (!req->submitted) { + if (!req->submitted && req->state < HTTP_REQUEST_STATE_FINISHED) { /* we're still in http_client_request_submit(). delay reporting the error, so the caller doesn't have to handle immediate callbacks. */ diff -r b4d54b6f3d10 -r a8fe88d2286d src/lib-http/http-client.h --- a/src/lib-http/http-client.h Sat Oct 12 10:55:38 2013 +0300 +++ b/src/lib-http/http-client.h Sat Oct 12 10:57:05 2013 +0300 @@ -16,6 +16,7 @@ HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, + HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, }; From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: Implemented functions for cloning and cop... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9709839d2be7 changeset: 16853:9709839d2be7 user: Stephan Bosch date: Sat Oct 12 10:58:04 2013 +0300 description: lib-http: Implemented functions for cloning and copying HTTP URLs and for constructing partial URLs. diffstat: src/lib-http/http-url.c | 109 +++++++++++++++++++++++++++++++++++++++-------- src/lib-http/http-url.h | 12 +++++ 2 files changed, 102 insertions(+), 19 deletions(-) diffs (163 lines): diff -r a8fe88d2286d -r 9709839d2be7 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Sat Oct 12 10:57:05 2013 +0300 +++ b/src/lib-http/http-url.c Sat Oct 12 10:58:04 2013 +0300 @@ -418,10 +418,78 @@ } /* + * HTTP URL manipulation + */ + +void http_url_copy_authority(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + dest->host_name = p_strdup(pool, src->host_name); + if (src->have_host_ip) { + dest->host_ip = src->host_ip; + dest->have_host_ip = TRUE; + } + if (src->have_port) { + dest->port = src->port; + dest->have_port = TRUE; + } + dest->have_ssl = src->have_ssl; +} + +void http_url_copy(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + http_url_copy_authority(pool, dest, src); + dest->path = p_strdup(pool, src->path); + dest->enc_query = p_strdup(pool, src->enc_query); + dest->enc_fragment = p_strdup(pool, src->enc_fragment); +} + +struct http_url *http_url_clone(pool_t pool,const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy(pool, new_url, src); + + return new_url; +} + +/* * HTTP URL construction */ -static void http_url_add_target(string_t *urlstr, const struct http_url *url) +static void +http_url_add_scheme(string_t *urlstr, const struct http_url *url) +{ + /* scheme */ + if (!url->have_ssl) + uri_append_scheme(urlstr, "http"); + else + uri_append_scheme(urlstr, "https"); + str_append(urlstr, "//"); +} + +static void +http_url_add_authority(string_t *urlstr, const struct http_url *url) +{ + /* host:port */ + if (url->host_name != NULL) { + /* assume IPv6 literal if starts with '['; avoid encoding */ + if (*url->host_name == '[') + str_append(urlstr, url->host_name); + else + uri_append_host_name(urlstr, url->host_name); + } else if (url->have_host_ip) { + uri_append_host_ip(urlstr, &url->host_ip); + } else + i_unreached(); + if (url->have_port) + uri_append_port(urlstr, url->port); +} + +static void +http_url_add_target(string_t *urlstr, const struct http_url *url) { if (url->path == NULL || *url->path == '\0') { /* Older syntax of RFC 2616 requires this slash at all times for an @@ -443,24 +511,8 @@ { string_t *urlstr = t_str_new(512); - /* scheme */ - uri_append_scheme(urlstr, "http"); - str_append(urlstr, "//"); - - /* host:port */ - if (url->host_name != NULL) { - /* assume IPv6 literal if starts with '['; avoid encoding */ - if (*url->host_name == '[') - str_append(urlstr, url->host_name); - else - uri_append_host_name(urlstr, url->host_name); - } else if (url->have_host_ip) { - uri_append_host_ip(urlstr, &url->host_ip); - } else - i_unreached(); - if (url->have_port) - uri_append_port(urlstr, url->port); - + http_url_add_scheme(urlstr, url); + http_url_add_authority(urlstr, url); http_url_add_target(urlstr, url); /* fragment */ @@ -472,6 +524,25 @@ return str_c(urlstr); } +const char *http_url_create_host(const struct http_url *url) +{ + string_t *urlstr = t_str_new(512); + + http_url_add_scheme(urlstr, url); + http_url_add_authority(urlstr, url); + + return str_c(urlstr); +} + +const char *http_url_create_authority(const struct http_url *url) +{ + string_t *urlstr = t_str_new(256); + + http_url_add_authority(urlstr, url); + + return str_c(urlstr); +} + const char *http_url_create_target(const struct http_url *url) { string_t *urlstr = t_str_new(256); diff -r a8fe88d2286d -r 9709839d2be7 src/lib-http/http-url.h --- a/src/lib-http/http-url.h Sat Oct 12 10:57:05 2013 +0300 +++ b/src/lib-http/http-url.h Sat Oct 12 10:58:04 2013 +0300 @@ -52,11 +52,23 @@ struct http_request_target *target, const char **error_r); /* + * HTTP URL manipulation + */ + +void http_url_copy_authority(pool_t pool, struct http_url *dest, + const struct http_url *src); +void http_url_copy(pool_t pool, struct http_url *dest, + const struct http_url *src); +struct http_url *http_url_clone(pool_t pool,const struct http_url *src); + +/* * HTTP URL construction */ const char *http_url_create(const struct http_url *url); +const char *http_url_create_host(const struct http_url *url); +const char *http_url_create_authority(const struct http_url *url); const char *http_url_create_target(const struct http_url *url); void http_url_escape_param(string_t *out, const char *data); From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: Implemented proxy support. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/462ae2cb8094 changeset: 16854:462ae2cb8094 user: Stephan Bosch date: Sat Oct 12 11:00:15 2013 +0300 description: lib-http: http-client: Implemented proxy support. diffstat: src/lib-http/http-client-host.c | 14 ++- src/lib-http/http-client-private.h | 34 ++++++++- src/lib-http/http-client-request.c | 125 ++++++++++++++++++++++++++---------- src/lib-http/http-client.c | 9 ++ src/lib-http/http-client.h | 16 ++++ 5 files changed, 151 insertions(+), 47 deletions(-) diffs (truncated from 394 to 300 lines): diff -r 9709839d2be7 -r 462ae2cb8094 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Oct 12 10:58:04 2013 +0300 +++ b/src/lib-http/http-client-host.c Sat Oct 12 11:00:15 2013 +0300 @@ -454,12 +454,14 @@ struct http_client_request *req) { struct http_client_host_port *hport; - const char *https_name = req->ssl ? req->hostname : NULL; + const struct http_url *host_url = req->host_url; + const char *https_name = http_client_request_https_name(req); + in_port_t port = http_client_request_port(req); const char *error; req->host = host; - if (req->ssl && host->client->ssl_ctx == NULL) { + if (host_url->have_ssl && host->client->ssl_ctx == NULL) { if (http_client_init_ssl_ctx(host->client, &error) < 0) { http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, error); @@ -468,7 +470,7 @@ } /* add request to host (grouped by tcp port) */ - hport = http_client_host_port_init(host, req->port, https_name); + hport = http_client_host_port_init(host, port, https_name); if (req->urgent) array_insert(&hport->request_queue, 0, &req, 1); else @@ -542,9 +544,10 @@ struct http_client_request *req) { struct http_client_host_port *hport; - const char *https_name = req->ssl ? req->hostname : NULL; + const char *https_name = http_client_request_https_name(req); + in_port_t port = http_client_request_port(req); - hport = http_client_host_port_find(host, req->port, https_name); + hport = http_client_host_port_find(host, port, https_name); if (hport == NULL) return; @@ -596,5 +599,4 @@ (*req)->to_delayed_error = io_loop_move_timeout(&(*req)->to_delayed_error); } - } diff -r 9709839d2be7 -r 462ae2cb8094 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Oct 12 10:58:04 2013 +0300 +++ b/src/lib-http/http-client-private.h Sat Oct 12 11:00:15 2013 +0300 @@ -3,6 +3,7 @@ #include "connection.h" +#include "http-url.h" #include "http-client.h" #define HTTP_DEFAULT_PORT 80 @@ -38,9 +39,13 @@ struct http_client_request { pool_t pool; unsigned int refcount; + const char *label; - const char *method, *hostname, *target; - in_port_t port; + const char *method, *target; + struct http_url origin_url; + + const struct http_url *host_url; + const char *authority; struct http_client *client; struct http_client_host *host; @@ -79,7 +84,6 @@ unsigned int payload_sync:1; unsigned int payload_chunked:1; unsigned int payload_wait:1; - unsigned int ssl:1; unsigned int urgent:1; unsigned int submitted:1; }; @@ -210,8 +214,28 @@ static inline const char * http_client_request_label(struct http_client_request *req) { - return t_strdup_printf("[%s http%s://%s:%d%s]", - req->method, req->ssl ? "s" : "", req->hostname, req->port, req->target); + if (req->label == NULL) { + return t_strdup_printf("[%s %s%s]", + req->method, http_url_create(&req->origin_url), req->target); + } + return req->label; +} + +static inline in_port_t +http_client_request_port(const struct http_client_request *req) +{ + const struct http_url *host_url = req->host_url; + + return (host_url->have_port ? host_url->port : + (host_url->have_ssl ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT)); +} + +static inline const char * +http_client_request_https_name(const struct http_client_request *req) +{ + const struct http_url *host_url = req->host_url; + + return (host_url->have_ssl ? host_url->host_name : NULL); } static inline const char * diff -r 9709839d2be7 -r 462ae2cb8094 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Oct 12 10:58:04 2013 +0300 +++ b/src/lib-http/http-client-request.c Sat Oct 12 11:00:15 2013 +0300 @@ -52,10 +52,8 @@ */ static void http_client_request_remove_delayed(struct http_client_request *req); -#undef http_client_request -struct http_client_request * -http_client_request(struct http_client *client, - const char *method, const char *host, const char *target, +static struct http_client_request * +http_client_request_new(struct http_client *client, const char *method, http_client_request_callback_t *callback, void *context) { pool_t pool; @@ -67,18 +65,44 @@ req->refcount = 1; req->client = client; req->method = p_strdup(pool, method); - req->hostname = p_strdup(pool, host); - req->port = HTTP_DEFAULT_PORT; - req->target = (target == NULL ? "/" : p_strdup(pool, target)); req->callback = callback; req->context = context; - req->headers = str_new(default_pool, 256); req->date = (time_t)-1; req->state = HTTP_REQUEST_STATE_NEW; return req; } +#undef http_client_request +struct http_client_request * +http_client_request(struct http_client *client, + const char *method, const char *host, const char *target, + http_client_request_callback_t *callback, void *context) +{ + struct http_client_request *req; + + req = http_client_request_new(client, method, callback, context); + req->origin_url.host_name = p_strdup(req->pool, host); + req->target = (target == NULL ? "/" : p_strdup(req->pool, target)); + req->headers = str_new(default_pool, 256); + return req; +} + +#undef http_client_request_url +struct http_client_request * +http_client_request_url(struct http_client *client, + const char *method, const struct http_url *target_url, + http_client_request_callback_t *callback, void *context) +{ + struct http_client_request *req; + + req = http_client_request_new(client, method, callback, context); + http_url_copy_authority(req->pool, &req->origin_url, target_url); + req->target = p_strdup(req->pool, http_url_create_target(target_url)); + req->headers = str_new(default_pool, 256); + return req; +} + void http_client_request_ref(struct http_client_request *req) { req->refcount++; @@ -124,21 +148,15 @@ in_port_t port) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); - req->port = port; + req->origin_url.port = port; + req->origin_url.have_port = TRUE; } void http_client_request_set_ssl(struct http_client_request *req, bool ssl) { i_assert(req->state == HTTP_REQUEST_STATE_NEW); - if (ssl) { - if (!req->ssl && req->port == HTTP_DEFAULT_PORT) - req->port = HTTPS_DEFAULT_PORT; - } else { - if (req->ssl && req->port == HTTPS_DEFAULT_PORT) - req->port = HTTP_DEFAULT_PORT; - } - req->ssl = ssl; + req->origin_url.have_ssl = ssl; } void http_client_request_set_urgent(struct http_client_request *req) @@ -180,6 +198,8 @@ req->have_hdr_user_agent = TRUE; break; } + if (req->headers == NULL) + req->headers = str_new(default_pool, 256); str_printfa(req->headers, "%s: %s\r\n", key, value); } @@ -223,15 +243,39 @@ static void http_client_request_do_submit(struct http_client_request *req) { + struct http_client *client = req->client; struct http_client_host *host; + const struct http_url *proxy_url = client->set.proxy_url; + const char *target; i_assert(req->state == HTTP_REQUEST_STATE_NEW); + target = t_strconcat + (http_url_create_host(&req->origin_url), req->target, NULL); + + /* determine what host to contact to submit this request */ + if (proxy_url != NULL) { + req->host_url = proxy_url; /* proxy server */ + } else { + req->host_url = &req->origin_url; /* origin server */ + } + /* use submission date if no date is set explicitly */ if (req->date == (time_t)-1) req->date = ioloop_time; - host = http_client_host_get(req->client, req->hostname); + /* prepare value for Host header */ + req->authority = + p_strdup(req->pool, http_url_create_authority(req->host_url)); + + /* debug label */ + req->label = p_strdup_printf(req->pool, "[%s %s]", req->method, target); + + /* request target needs to be made absolute url for proxy requests */ + if (proxy_url != NULL) + req->target = p_strdup(req->pool, target); + + host = http_client_host_get(req->client, req->host_url->host_name); req->state = HTTP_REQUEST_STATE_QUEUED; http_client_host_submit_request(host, req); @@ -239,10 +283,10 @@ void http_client_request_submit(struct http_client_request *req) { - http_client_request_debug(req, "Submitted"); req->client->pending_requests++; http_client_request_do_submit(req); + http_client_request_debug(req, "Submitted"); req->submitted = TRUE; } @@ -452,11 +496,7 @@ http_client_request_add_header() */ if (!req->have_hdr_host) { str_append(rtext, "Host: "); - str_append(rtext, req->hostname); - if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) || - (req->ssl && req->port != HTTPS_DEFAULT_PORT)) { - str_printfa(rtext, ":%u", req->port); - } + str_append(rtext, req->authority); str_append(rtext, "\r\n"); } if (!req->have_hdr_date) { @@ -487,8 +527,16 @@ req->payload_output = output; o_stream_ref(output); } - if (!req->have_hdr_connection) + if (!req->have_hdr_connection && req->host_url == &req->origin_url) { + /* https://tools.ietf.org/html/rfc2068 + Section 19.7.1: + + A client MUST NOT send the Keep-Alive connection token to a proxy + server as HTTP/1.0 proxy servers do not obey the rules of HTTP/1.1 + for parsing the Connection header field. + */ str_append(rtext, "Connection: Keep-Alive\r\n"); + } /* request line + implicit headers */ iov[0].iov_base = str_data(rtext); @@ -665,8 +713,7 @@ unsigned int status, const char *location) { From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: Added support for creating CONNECT tunnel... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c3884a6acde6 changeset: 16855:c3884a6acde6 user: Stephan Bosch date: Sat Oct 12 11:05:08 2013 +0300 description: lib-http: Added support for creating CONNECT tunnels through HTTP. diffstat: src/lib-http/http-client-connection.c | 131 +++++++++++++++++++++++++++------ src/lib-http/http-client-host.c | 54 +++++++------ src/lib-http/http-client-peer.c | 27 +++++- src/lib-http/http-client-private.h | 60 +++++++++++--- src/lib-http/http-client-request.c | 89 +++++++++++++++++++--- src/lib-http/http-client.h | 27 +++++++ src/lib-http/http-response-parser.c | 10 +- src/lib-http/http-response-parser.h | 4 +- src/lib-http/http-response.h | 6 + 9 files changed, 319 insertions(+), 89 deletions(-) diffs (truncated from 918 to 300 lines): diff -r 462ae2cb8094 -r c3884a6acde6 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Oct 12 11:00:15 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sat Oct 12 11:05:08 2013 +0300 @@ -57,7 +57,7 @@ bool http_client_connection_is_ready(struct http_client_connection *conn) { return (conn->connected && !conn->output_locked && - !conn->close_indicated && + !conn->close_indicated && !conn->tunneling && http_client_connection_count_pending(conn) < conn->client->set.max_pipelined_requests); } @@ -173,7 +173,8 @@ { unsigned int timeout, count; - if (array_count(&conn->request_wait_list) == 0 && + if (array_is_created(&conn->request_wait_list) && + array_count(&conn->request_wait_list) == 0 && conn->incoming_payload == NULL && conn->client->set.max_idle_time_msecs > 0) { @@ -302,6 +303,9 @@ return -1; } + if (req->connect_tunnel) + conn->tunneling = TRUE; + /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21; Section 6.1.2.1: @@ -388,6 +392,7 @@ { struct http_client_connection *conn = req->conn; + i_assert(conn != NULL); i_assert(conn->pending_request == req); i_assert(conn->incoming_payload != NULL); i_assert(conn->conn.io == NULL); @@ -479,7 +484,8 @@ } if (conn->incoming_payload == NULL) { - i_assert(conn->conn.io != NULL); + i_assert(conn->conn.io != NULL || + conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW); return TRUE; } @@ -495,7 +501,7 @@ struct http_client_request *req = NULL; int finished = 0, ret; const char *error; - bool no_payload = FALSE; + enum http_response_payload_type payload_type; i_assert(conn->incoming_payload == NULL); @@ -516,18 +522,16 @@ req_idx = array_idx(&conn->request_wait_list, 0); req = req_idx[0]; - /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21 - Section 3.3.2: - - A server MAY send a Content-Length header field in a response to a - HEAD request [...] - */ - no_payload = (strcmp(req->method, "HEAD") == 0); + /* determine whether to expect a response payload */ + payload_type = http_client_request_get_payload_type(req); + } else { + req = NULL; + payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; } // FIXME: handle somehow if server replies before request->input is at EOF while ((ret=http_response_parse_next - (conn->http_parser, no_payload, &response, &error)) > 0) { + (conn->http_parser, payload_type, &response, &error)) > 0) { bool aborted; if (req == NULL) { @@ -621,13 +625,15 @@ if (array_count(&conn->request_wait_list) > 0) { req_idx = array_idx(&conn->request_wait_list, 0); req = req_idx[0]; - no_payload = (strcmp(req->method, "HEAD") == 0); + + /* determine whether to expect a response payload */ + payload_type = http_client_request_get_payload_type(req); } else { /* no more requests waiting for the connection */ if (conn->to_requests != NULL) timeout_remove(&conn->to_requests); req = NULL; - no_payload = FALSE; + payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; } } @@ -702,6 +708,33 @@ return 1; } +void +http_client_connection_start_tunnel(struct http_client_connection **_conn, + struct http_client_tunnel *tunnel) +{ + struct http_client_connection *conn = *_conn; + + i_assert(conn->tunneling); + + /* claim connection streams */ + memset(tunnel, 0, sizeof(*tunnel)); + tunnel->input = conn->conn.input; + tunnel->output = conn->conn.output; + tunnel->fd_in = conn->conn.fd_in; + tunnel->fd_out = conn->conn.fd_out; + + /* detach from connection */ + conn->conn.input = NULL; + conn->conn.output = NULL; + conn->conn.fd_in = -1; + conn->conn.fd_out = -1; + conn->closing = TRUE; + conn->connected = FALSE; + connection_disconnect(&conn->conn); + + http_client_connection_unref(_conn); +} + static void http_client_connection_ready(struct http_client_connection *conn) { @@ -725,6 +758,34 @@ &conn->conn.input, &conn->conn.output); } + /* direct tunneling connections handle connect requests just by providing a + raw connection */ + if (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) { + struct http_client_request *req; + + req = http_client_peer_claim_request(conn->peer, FALSE); + if (req != NULL) { + struct http_response response; + + http_client_request_ref(req); + req->conn = conn; + conn->tunneling = TRUE; + + memset(&response, 0, sizeof(response)); + response.status = 200; + response.reason = "OK"; + + (void)http_client_connection_return_response(conn, req, &response); + http_client_request_unref(&req); + return; + } + + http_client_connection_debug(conn, + "No raw connect requests pending; closing useless connection"); + http_client_connection_unref(&conn); + return; + } + /* start protocol I/O */ conn->http_parser = http_response_parser_init (conn->conn.input, &conn->client->set.response_hdr_limits); @@ -881,13 +942,27 @@ struct http_client_connection *conn; static unsigned int id = 0; const struct http_client_peer_addr *addr = &peer->addr; + const char *conn_type = "UNKNOWN"; + + switch (peer->addr.type) { + case HTTP_CLIENT_PEER_ADDR_HTTP: + conn_type = "HTTP"; + break; + case HTTP_CLIENT_PEER_ADDR_HTTPS: + conn_type = "HTTPS"; + break; + case HTTP_CLIENT_PEER_ADDR_RAW: + conn_type = "Raw"; + break; + } conn = i_new(struct http_client_connection, 1); conn->refcount = 1; conn->client = peer->client; conn->id = id++; conn->peer = peer; - i_array_init(&conn->request_wait_list, 16); + if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW) + i_array_init(&conn->request_wait_list, 16); connection_init_client_ip (peer->client->conn_list, &conn->conn, &addr->ip, addr->port); @@ -896,8 +971,9 @@ array_append(&peer->conns, &conn, 1); http_client_connection_debug(conn, - "Connection created (%d parallel connections exist)%s", - array_count(&peer->conns), (conn->to_input == NULL ? "" : " [broken]")); + "%s connection created (%d parallel connections exist)%s", + conn_type, array_count(&peer->conns), + (conn->to_input == NULL ? "" : " [broken]")); return conn; } @@ -910,6 +986,7 @@ { struct http_client_connection *conn = *_conn; struct http_client_connection *const *conn_idx; + ARRAY_TYPE(http_client_connection) *conn_arr; struct http_client_peer *peer = conn->peer; struct http_client_request **req; @@ -931,17 +1008,19 @@ connection_disconnect(&conn->conn); - /* abort all pending requests */ - array_foreach_modifiable(&conn->request_wait_list, req) { - i_assert((*req)->submitted); - http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, - "Aborting"); + if (array_is_created(&conn->request_wait_list)) { + /* abort all pending requests */ + array_foreach_modifiable(&conn->request_wait_list, req) { + i_assert((*req)->submitted); + http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Aborting"); + } + array_free(&conn->request_wait_list); } if (conn->pending_request != NULL) { http_client_request_error(conn->pending_request, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborting"); } - array_free(&conn->request_wait_list); if (conn->http_parser != NULL) http_response_parser_deinit(&conn->http_parser); @@ -964,10 +1043,10 @@ timeout_remove(&conn->to_response); /* remove this connection from the list */ - array_foreach(&conn->peer->conns, conn_idx) { + conn_arr = &conn->peer->conns; + array_foreach(conn_arr, conn_idx) { if (*conn_idx == conn) { - array_delete(&conn->peer->conns, - array_foreach_idx(&conn->peer->conns, conn_idx), 1); + array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1); break; } } diff -r 462ae2cb8094 -r c3884a6acde6 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Oct 12 11:00:15 2013 +0300 +++ b/src/lib-http/http-client-host.c Sat Oct 12 11:05:08 2013 +0300 @@ -46,13 +46,13 @@ static struct http_client_host_port * http_client_host_port_find(struct http_client_host *host, - in_port_t port, const char *https_name) + const struct http_client_peer_addr *addr) { struct http_client_host_port *hport; array_foreach_modifiable(&host->ports, hport) { - if (hport->addr.port == port && - null_strcmp(hport->addr.https_name, https_name) == 0) + if (hport->addr.type == addr->type && hport->addr.port == addr->port && + null_strcmp(hport->addr.https_name, addr->https_name) == 0) return hport; } @@ -61,16 +61,17 @@ static struct http_client_host_port * http_client_host_port_init(struct http_client_host *host, - in_port_t port, const char *https_name) + const struct http_client_peer_addr *addr) { struct http_client_host_port *hport; - hport = http_client_host_port_find(host, port, https_name); + hport = http_client_host_port_find(host, addr); if (hport == NULL) { hport = array_append_space(&host->ports); hport->host = host; - hport->addr.port = port; - hport->addr.https_name = i_strdup(https_name); + hport->addr = *addr; + hport->https_name = i_strdup(addr->https_name); + hport->addr.https_name = hport->https_name; hport->ips_connect_idx = 0; From dovecot at dovecot.org Sat Oct 12 11:14:58 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 12 Oct 2013 11:14:58 +0300 Subject: dovecot-2.2: lib-http: http-client: Added support for tunneling ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9498573f0d0 changeset: 16856:b9498573f0d0 user: Stephan Bosch date: Sat Oct 12 11:11:04 2013 +0300 description: lib-http: http-client: Added support for tunneling SSL conntections through proxy. diffstat: src/lib-http/http-client-connection.c | 108 ++++++++++++++++++++++++++++++++- src/lib-http/http-client-host.c | 9 ++- src/lib-http/http-client-peer.c | 1 + src/lib-http/http-client-private.h | 21 +++++- src/lib-http/http-client-request.c | 31 ++++++++- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 14 ++++ 7 files changed, 172 insertions(+), 13 deletions(-) diffs (truncated from 361 to 300 lines): diff -r c3884a6acde6 -r b9498573f0d0 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Oct 12 11:05:08 2013 +0300 +++ b/src/lib-http/http-client-connection.c Sat Oct 12 11:11:04 2013 +0300 @@ -4,6 +4,7 @@ #include "net.h" #include "str.h" #include "hash.h" +#include "llist.h" #include "array.h" #include "ioloop.h" #include "istream.h" @@ -912,7 +913,8 @@ http_client_connection_destroy(&conn->conn); } -static void http_client_connection_connect(struct http_client_connection *conn) +static void +http_client_connection_connect(struct http_client_connection *conn) { unsigned int msecs; @@ -936,6 +938,94 @@ } } +static void +http_client_connect_tunnel_timeout(struct http_client_connection *conn) +{ + http_client_connection_unref(&conn); +} + +// FIXME: put something like this in lib/connection.c +static void +_connection_init_from_streams(struct connection_list *list, + struct connection *conn, const char *name, + struct istream *input, struct ostream *output) +{ + i_assert(name != NULL); + + conn->list = list; + conn->name = i_strdup(name); + conn->fd_in = i_stream_get_fd(input); + conn->fd_out = o_stream_get_fd(output); + + i_assert(conn->fd_in >= 0); + i_assert(conn->fd_out >= 0); + i_assert(conn->io == NULL); + i_assert(conn->input == NULL); + i_assert(conn->output == NULL); + i_assert(conn->to == NULL); + + conn->input = input; + i_stream_set_name(conn->input, conn->name); + + conn->output = output; + o_stream_set_no_error_handling(conn->output, TRUE); + o_stream_set_name(conn->output, conn->name); + + conn->io = io_add(conn->fd_in, IO_READ, *list->v.input, conn); + + DLLIST_PREPEND(&list->connections, conn); + list->connections_count++; + + if (list->v.client_connected != NULL) + list->v.client_connected(conn, TRUE); +} + +static void +http_client_connection_tunnel_response(const struct http_response *response, + struct http_client_connection *conn) +{ + struct http_client_tunnel tunnel; + const char *name = http_client_peer_addr2str(&conn->peer->addr); + + if (response->status != 200) { + http_client_peer_connection_failure(conn->peer, t_strdup_printf( + "tunnel connect(%s) failed: %d %s", name, + response->status, response->reason)); + conn->connect_request = NULL; + return; + } + + http_client_request_start_tunnel(conn->connect_request, &tunnel); + conn->connect_request = NULL; + + _connection_init_from_streams + (conn->client->conn_list, &conn->conn, name, tunnel.input, tunnel.output); +} + +static void +http_client_connection_connect_tunnel(struct http_client_connection *conn, + const struct ip_addr *ip, unsigned int port) +{ + unsigned int msecs; + + conn->connect_start_timestamp = ioloop_timeval; + + conn->connect_request = http_client_request_connect_ip + (conn->client, ip, port, http_client_connection_tunnel_response, conn); + http_client_request_set_urgent(conn->connect_request); + http_client_request_submit(conn->connect_request); + + /* don't use connection.h timeout because we want this timeout + to include also the SSL handshake */ + msecs = conn->client->set.connect_timeout_msecs; + if (msecs == 0) + msecs = conn->client->set.request_timeout_msecs; + if (msecs > 0) { + conn->to_connect = + timeout_add(msecs, http_client_connect_tunnel_timeout, conn); + } +} + struct http_client_connection * http_client_connection_create(struct http_client_peer *peer) { @@ -951,6 +1041,9 @@ case HTTP_CLIENT_PEER_ADDR_HTTPS: conn_type = "HTTPS"; break; + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + conn_type = "Tunneled HTTPS"; + break; case HTTP_CLIENT_PEER_ADDR_RAW: conn_type = "Raw"; break; @@ -964,9 +1057,13 @@ if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW) i_array_init(&conn->request_wait_list, 16); - connection_init_client_ip - (peer->client->conn_list, &conn->conn, &addr->ip, addr->port); - http_client_connection_connect(conn); + if (peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) { + http_client_connection_connect_tunnel(conn, &addr->ip, addr->port); + } else { + connection_init_client_ip + (peer->client->conn_list, &conn->conn, &addr->ip, addr->port); + http_client_connection_connect(conn); + } array_append(&peer->conns, &conn, 1); @@ -1000,6 +1097,9 @@ conn->closing = TRUE; conn->connected = FALSE; + if (conn->connect_request != NULL) + http_client_request_abort(&conn->connect_request); + if (conn->incoming_payload != NULL) { /* the stream is still accessed by lib-http caller. */ i_stream_remove_destroy_callback(conn->incoming_payload, diff -r c3884a6acde6 -r b9498573f0d0 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Oct 12 11:05:08 2013 +0300 +++ b/src/lib-http/http-client-host.c Sat Oct 12 11:11:04 2013 +0300 @@ -428,9 +428,10 @@ } struct http_client_host *http_client_host_get -(struct http_client *client, const char *hostname) +(struct http_client *client, const struct http_url *host_url) { struct http_client_host *host; + const char *hostname = host_url->host_name; host = hash_table_lookup(client->hosts, hostname); if (host == NULL) { @@ -445,6 +446,12 @@ hash_table_insert(client->hosts, hostname, host); DLLIST_PREPEND(&client->hosts_list, host); + if (host_url->have_host_ip) { + host->ips_count = 1; + host->ips = i_new(struct ip_addr, host->ips_count); + host->ips[0] = host_url->host_ip; + } + http_client_host_debug(host, "Host created"); } return host; diff -r c3884a6acde6 -r b9498573f0d0 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Sat Oct 12 11:05:08 2013 +0300 +++ b/src/lib-http/http-client-peer.c Sat Oct 12 11:11:04 2013 +0300 @@ -48,6 +48,7 @@ case HTTP_CLIENT_PEER_ADDR_HTTP: return net_ip_hash(&peer->ip) + peer->port; case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: return net_ip_hash(&peer->ip) + peer->port + (peer->https_name == NULL ? 0 : str_hash(peer->https_name)); } diff -r c3884a6acde6 -r b9498573f0d0 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Oct 12 11:05:08 2013 +0300 +++ b/src/lib-http/http-client-private.h Sat Oct 12 11:11:04 2013 +0300 @@ -35,6 +35,7 @@ enum http_client_peer_addr_type { HTTP_CLIENT_PEER_ADDR_HTTP = 0, HTTP_CLIENT_PEER_ADDR_HTTPS, + HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL, HTTP_CLIENT_PEER_ADDR_RAW }; @@ -97,6 +98,7 @@ unsigned int submitted:1; unsigned int connect_tunnel:1; unsigned int connect_direct:1; + unsigned int ssl_tunnel:1; }; struct http_client_host_port { @@ -178,6 +180,7 @@ int connect_errno; struct timeval connect_start_timestamp; struct timeval connected_timestamp; + struct http_client_request *connect_request; struct ssl_iostream *ssl_iostream; struct http_response_parser *http_parser; @@ -248,7 +251,10 @@ addr->type = HTTP_CLIENT_PEER_ADDR_RAW; addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT); } else if (host_url->have_ssl) { - addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS; + if (req->ssl_tunnel) + addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL; + else + addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS; addr->https_name = host_url->host_name; addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT); } else { @@ -260,13 +266,19 @@ static inline const char * http_client_connection_label(struct http_client_connection *conn) { - return t_strdup_printf("%s [%d]", - http_client_peer_addr2str(&conn->peer->addr), conn->id); + return t_strdup_printf("%s%s [%d]", + http_client_peer_addr2str(&conn->peer->addr), + (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL ? + " (tunnel)" : ""), conn->id); } static inline const char * http_client_peer_label(struct http_client_peer *peer) { + if (peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) { + return t_strconcat + (http_client_peer_addr2str(&peer->addr), " (tunnel)", NULL); + } return http_client_peer_addr2str(&peer->addr); } @@ -344,7 +356,8 @@ void http_client_peer_switch_ioloop(struct http_client_peer *peer); struct http_client_host * - http_client_host_get(struct http_client *client, const char *hostname); +http_client_host_get(struct http_client *client, + const struct http_url *host_url); void http_client_host_free(struct http_client_host **_host); void http_client_host_submit_request(struct http_client_host *host, struct http_client_request *req); diff -r c3884a6acde6 -r b9498573f0d0 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Sat Oct 12 11:05:08 2013 +0300 +++ b/src/lib-http/http-client-request.c Sat Oct 12 11:11:04 2013 +0300 @@ -115,7 +115,24 @@ req->origin_url.port = port; req->origin_url.have_port = TRUE; req->connect_tunnel = TRUE; - req->target = ""; + req->target = req->origin_url.host_name; + return req; +} + +#undef http_client_request_connect_ip +struct http_client_request * +http_client_request_connect_ip(struct http_client *client, + const struct ip_addr *ip, in_port_t port, + http_client_request_callback_t *callback, + void *context) +{ + struct http_client_request *req; + const char *hostname = net_ip2addr(ip); + + req = http_client_request_connect + (client, hostname, port, callback, context); + req->origin_url.host_ip = *ip; + req->origin_url.have_host_ip = TRUE; return req; } @@ -300,9 +317,15 @@ /* determine what host to contact to submit this request */ if (proxy_url != NULL) { - req->host_url = proxy_url; /* proxy server */ + if (req->origin_url.have_ssl && !client->set.no_ssl_tunnel && + !req->connect_tunnel) { + req->host_url = &req->origin_url; /* tunnel to origin server */ From dovecot at dovecot.org Wed Oct 16 20:17:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 16 Oct 2013 20:17:25 +0300 Subject: dovecot-2.2: imap: Don't assert-crash on FETCH BODY[MIME] (retur... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/73216464c8e1 changeset: 16857:73216464c8e1 user: Timo Sirainen date: Wed Oct 16 20:16:45 2013 +0300 description: imap: Don't assert-crash on FETCH BODY[MIME] (return BAD instead) diffstat: src/lib-imap-storage/imap-msgpart.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r b9498573f0d0 -r 73216464c8e1 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Sat Oct 12 11:11:04 2013 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Wed Oct 16 20:16:45 2013 +0300 @@ -254,6 +254,8 @@ section = t_str_ucase(section); if (strcmp(section, "MIME") == 0) { + if (msgpart->section_number[0] == '\0') + return -1; msgpart->fetch_type = FETCH_MIME; msgpart->wanted_fields |= MAIL_FETCH_STREAM_BODY; } else if (strcmp(section, "TEXT") == 0) { From dovecot at dovecot.org Mon Oct 21 20:48:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Oct 2013 20:48:08 +0300 Subject: dovecot-2.2: imap: COPY allows transaction commit now to fail wi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bd65a83a9a3 changeset: 16858:7bd65a83a9a3 user: Timo Sirainen date: Mon Oct 21 20:47:53 2013 +0300 description: imap: COPY allows transaction commit now to fail with "some messages were expunged". diffstat: src/imap/cmd-copy.c | 12 +++++++++--- 1 files changed, 9 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 73216464c8e1 -r 7bd65a83a9a3 src/imap/cmd-copy.c --- a/src/imap/cmd-copy.c Wed Oct 16 20:16:45 2013 +0300 +++ b/src/imap/cmd-copy.c Mon Oct 21 20:47:53 2013 +0300 @@ -127,9 +127,15 @@ msg = t_str_new(256); if (ret <= 0) mailbox_transaction_rollback(&t); - else if (mailbox_transaction_commit_get_changes(&t, &changes) < 0) - ret = -1; - else if (copy_count == 0) { + else if (mailbox_transaction_commit_get_changes(&t, &changes) < 0) { + if (mailbox_get_last_mail_error(destbox) == MAIL_ERROR_EXPUNGED) { + /* storage backend didn't notice the expunge until + at commit time. */ + ret = 0; + } else { + ret = -1; + } + } else if (copy_count == 0) { str_append(msg, "OK No messages found."); pool_unref(&changes.pool); } else if (seq_range_count(&changes.saved_uids) == 0 || From dovecot at dovecot.org Mon Oct 21 21:38:25 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 21 Oct 2013 21:38:25 +0300 Subject: dovecot-2.2: doveadm backup: If -D parameter is given, log the r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cce994007aad changeset: 16859:cce994007aad user: Timo Sirainen date: Mon Oct 21 21:38:17 2013 +0300 description: doveadm backup: If -D parameter is given, log the reason why mailbox gets deleted. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree.c | 5 +- src/doveadm/dsync/dsync-mailbox-tree-sync.c | 68 +++++++++++++++++------ src/doveadm/dsync/dsync-mailbox-tree.h | 11 +++- src/doveadm/dsync/test-dsync-mailbox-tree-sync.c | 4 +- 4 files changed, 66 insertions(+), 22 deletions(-) diffs (255 lines): diff -r 7bd65a83a9a3 -r cce994007aad src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Oct 21 20:47:53 2013 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Oct 21 21:38:17 2013 +0300 @@ -268,6 +268,9 @@ struct dsync_mailbox_tree_sync_ctx *ctx; const struct dsync_mailbox_tree_sync_change *change; enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags = + (brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) | + (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0); if (brain->no_backup_overwrite) sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; @@ -280,7 +283,7 @@ ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree, brain->remote_mailbox_tree, - sync_type); + sync_type, sync_flags); while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { if (dsync_brain_mailbox_tree_sync_change(brain, change) < 0) brain->failed = TRUE; diff -r 7bd65a83a9a3 -r cce994007aad src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Mon Oct 21 20:47:53 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Mon Oct 21 21:38:17 2013 +0300 @@ -8,6 +8,7 @@ #include "hex-binary.h" #include "aqueue.h" #include "hash.h" +#include "dsync-brain-private.h" #include "dsync-mailbox-tree-private.h" #define TEMP_MAX_NAME_LEN 100 @@ -24,8 +25,10 @@ struct dsync_mailbox_tree_sync_ctx { pool_t pool; + struct dsync_brain *brain; struct dsync_mailbox_tree *local_tree, *remote_tree; enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags; ARRAY(struct dsync_mailbox_tree_sync_change) changes; unsigned int change_idx; @@ -144,11 +147,19 @@ static void sync_delete_mailbox_node(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_tree *tree, - struct dsync_mailbox_node *node) + struct dsync_mailbox_node *node, const char *reason) { struct dsync_mailbox_tree_sync_change *change; const char *name; + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + tree == ctx->local_tree) { + i_debug("brain %c: Deleting mailbox '%s' (GUID %s): %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + dsync_mailbox_node_get_full_name(tree, node), + guid_128_to_string(node->mailbox_guid), reason); + } + if (tree == ctx->local_tree) { /* delete this mailbox locally */ change = array_append_space(&ctx->changes); @@ -165,7 +176,7 @@ static void sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_tree *tree, - struct dsync_mailbox_node *node) + struct dsync_mailbox_node *node, const char *reason) { struct dsync_mailbox_tree *other_tree; struct dsync_mailbox_node *other_node; @@ -178,9 +189,9 @@ if (other_node == NULL) { /* doesn't exist / already deleted */ } else { - sync_delete_mailbox_node(ctx, other_tree, other_node); + sync_delete_mailbox_node(ctx, other_tree, other_node, reason); } - sync_delete_mailbox_node(ctx, tree, node); + sync_delete_mailbox_node(ctx, tree, node, reason); } static void @@ -206,7 +217,8 @@ if (!ignore_deletes) { /* this mailbox was deleted. delete it from the other side as well */ - sync_delete_mailbox(ctx, tree, node); + sync_delete_mailbox(ctx, tree, node, + "Mailbox has been deleted"); } else { /* we want to restore the mailbox back. just treat it as if it didn't exist */ @@ -904,18 +916,33 @@ } static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node, - const struct dsync_mailbox_node *wanted_node) + const struct dsync_mailbox_node *wanted_node, + const char **reason_r) { - if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) + if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + *reason_r = wanted_node->existence == DSYNC_MAILBOX_NODE_DELETED ? + "Mailbox has been deleted" : "Mailbox doesn't exist"; return TRUE; - if (node->uid_validity != wanted_node->uid_validity) + } + if (node->uid_validity != wanted_node->uid_validity) { + *reason_r = t_strdup_printf("UIDVALIDITY changed (%u -> %u)", + wanted_node->uid_validity, + node->uid_validity); return TRUE; + } if (node->uid_next > wanted_node->uid_next) { /* we can't lower the UIDNEXT */ + *reason_r = t_strdup_printf("UIDNEXT is too high (%u > %u)", + node->uid_next, + wanted_node->uid_next); return TRUE; } - return memcmp(node->mailbox_guid, wanted_node->mailbox_guid, - sizeof(node->mailbox_guid)) != 0; + if (memcmp(node->mailbox_guid, wanted_node->mailbox_guid, + sizeof(node->mailbox_guid)) != 0) { + *reason_r = "GUID changed"; + return TRUE; + } + return FALSE; } static void @@ -925,6 +952,7 @@ struct dsync_mailbox_node *node, const struct dsync_mailbox_node *wanted_node) { + const char *reason; int ret; while (node != NULL && wanted_node != NULL) { @@ -937,20 +965,22 @@ if (ret < 0) { /* node shouldn't exist */ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && - !dsync_mailbox_node_is_dir(node)) - sync_delete_mailbox_node(ctx, tree, node); + !dsync_mailbox_node_is_dir(node)) { + sync_delete_mailbox_node(ctx, tree, node, + "Mailbox doesn't exist"); + } node = node->next; } else if (ret > 0) { /* wanted_node doesn't exist. it's created later. */ wanted_node = wanted_node->next; - } else { - if (sync_is_wrong_mailbox(node, wanted_node) && + } else T_BEGIN { + if (sync_is_wrong_mailbox(node, wanted_node, &reason) && node->existence == DSYNC_MAILBOX_NODE_EXISTS && !dsync_mailbox_node_is_dir(node)) - sync_delete_mailbox_node(ctx, tree, node); + sync_delete_mailbox_node(ctx, tree, node, reason); node = node->next; wanted_node = wanted_node->next; - } + } T_END; } for (; node != NULL; node = node->next) { /* node and its children shouldn't exist */ @@ -961,7 +991,7 @@ } if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && !dsync_mailbox_node_is_dir(node)) - sync_delete_mailbox_node(ctx, tree, node); + sync_delete_mailbox_node(ctx, tree, node, "Mailbox doesn't exist"); } } @@ -1157,7 +1187,8 @@ struct dsync_mailbox_tree_sync_ctx * dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, struct dsync_mailbox_tree *remote_tree, - enum dsync_mailbox_trees_sync_type sync_type) + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags) { struct dsync_mailbox_tree_sync_ctx *ctx; pool_t pool; @@ -1173,6 +1204,7 @@ ctx->local_tree = local_tree; ctx->remote_tree = remote_tree; ctx->sync_type = sync_type; + ctx->sync_flags = sync_flags; i_array_init(&ctx->changes, 128); ignore_deletes = sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL; diff -r 7bd65a83a9a3 -r cce994007aad src/doveadm/dsync/dsync-mailbox-tree.h --- a/src/doveadm/dsync/dsync-mailbox-tree.h Mon Oct 21 20:47:53 2013 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree.h Mon Oct 21 21:38:17 2013 +0300 @@ -4,6 +4,7 @@ #include "guid.h" struct mail_namespace; +struct dsync_brain; enum dsync_mailbox_trees_sync_type { /* two-way sync for both mailboxes */ @@ -14,6 +15,13 @@ DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE }; +enum dsync_mailbox_trees_sync_flags { + /* Enable debugging */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG = 0x01, + /* Show ourself as "master brain" in the debug output */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02 +}; + enum dsync_mailbox_node_existence { /* this is just a filler node for children or for subscription deletion */ @@ -174,7 +182,8 @@ struct dsync_mailbox_tree_sync_ctx * dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, struct dsync_mailbox_tree *remote_tree, - enum dsync_mailbox_trees_sync_type sync_type); + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags); const struct dsync_mailbox_tree_sync_change * dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx); void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); diff -r 7bd65a83a9a3 -r cce994007aad src/doveadm/dsync/test-dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c Mon Oct 21 20:47:53 2013 +0300 +++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c Mon Oct 21 21:38:17 2013 +0300 @@ -170,7 +170,7 @@ dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2); dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2); ctx = dsync_mailbox_trees_sync_init(tree1, tree2, - DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0); while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { } dsync_mailbox_trees_sync_deinit(&ctx); @@ -185,7 +185,7 @@ dsync_mailbox_tree_build_guid_hash(orig_tree1, &dup_node1, &dup_node2); dsync_mailbox_tree_build_guid_hash(orig_tree2, &dup_node1, &dup_node2); ctx = dsync_mailbox_trees_sync_init(orig_tree2, orig_tree1, - DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0); while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { } dsync_mailbox_trees_sync_deinit(&ctx); From dovecot at dovecot.org Tue Oct 22 15:35:17 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Oct 2013 15:35:17 +0300 Subject: dovecot-2.2: lib-dns: Added alternative API for doing longer con... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a6abdf4b6222 changeset: 16860:a6abdf4b6222 user: Timo Sirainen date: Tue Oct 22 15:34:55 2013 +0300 description: lib-dns: Added alternative API for doing longer connections to dns-client process. diffstat: src/lib-dns/dns-lookup.c | 303 ++++++++++++++++++++++++++++++++-------------- src/lib-dns/dns-lookup.h | 26 ++++ 2 files changed, 237 insertions(+), 92 deletions(-) diffs (truncated from 432 to 300 lines): diff -r cce994007aad -r a6abdf4b6222 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Mon Oct 21 21:38:17 2013 +0300 +++ b/src/lib-dns/dns-lookup.c Tue Oct 22 15:34:55 2013 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "net.h" +#include "llist.h" #include "istream.h" #include "write-full.h" #include "time-util.h" @@ -14,12 +15,10 @@ #define MAX_INBUF_SIZE 512 struct dns_lookup { - int fd; - char *path; + struct dns_lookup *prev, *next; + struct dns_client *client; bool ptr_lookup; - struct istream *input; - struct io *io; struct timeout *to; struct timeval start_time; @@ -34,20 +33,51 @@ void *context; }; +struct dns_client { + int fd; + char *path; + + unsigned int timeout_msecs, idle_timeout_msecs; + + struct istream *input; + struct io *io; + struct timeout *to_idle; + + struct dns_lookup *head, *tail; + bool deinit_client_at_free; +}; + +#undef dns_lookup +#undef dns_lookup_ptr +#undef dns_client_lookup +#undef dns_client_lookup_ptr + static void dns_lookup_free(struct dns_lookup **_lookup); -static void dns_lookup_close(struct dns_lookup *lookup) +static void dns_client_disconnect(struct dns_client *client, const char *error) { - if (lookup->to != NULL) - timeout_remove(&lookup->to); - if (lookup->io != NULL) - io_remove(&lookup->io); - if (lookup->input != NULL) - i_stream_destroy(&lookup->input); - if (lookup->fd != -1) { - if (close(lookup->fd) < 0) - i_error("close(%s) failed: %m", lookup->path); - lookup->fd = -1; + struct dns_lookup *lookup; + struct dns_lookup_result result; + + memset(&result, 0, sizeof(result)); + result.ret = EAI_FAIL; + result.error = error; + + while (client->head != NULL) { + lookup = client->head; + lookup->callback(&result, lookup->context); + dns_lookup_free(&lookup); + } + if (client->to_idle != NULL) + timeout_remove(&client->to_idle); + if (client->io != NULL) + io_remove(&client->io); + if (client->input != NULL) + i_stream_destroy(&client->input); + if (client->fd != -1) { + if (close(client->fd) < 0) + i_error("close(%s) failed: %m", client->path); + client->fd = -1; } } @@ -107,129 +137,117 @@ lookup->result.msecs = diff; } -static void dns_lookup_input(struct dns_lookup *lookup) +static void dns_client_input(struct dns_client *client) { const char *line; - struct dns_lookup_result *result = &lookup->result; + struct dns_lookup *lookup = client->head; + bool retry = FALSE; int ret = 0; - while ((line = i_stream_read_next_line(lookup->input)) != NULL) { + while ((line = i_stream_read_next_line(client->input)) != NULL) { + if (lookup == NULL) { + dns_client_disconnect(client, t_strdup_printf( + "Unexpected input from %s", client->path)); + return; + } ret = dns_lookup_input_line(lookup, line); if (ret > 0) break; if (ret < 0) { - result->error = t_strdup_printf( - "Invalid input from %s", lookup->path); - break; + dns_client_disconnect(client, t_strdup_printf( + "Invalid input from %s", client->path)); + return; } } + if (ret == 0) + return; - if (result->error != NULL) { + if (lookup->result.error != NULL) { /* already got the error */ - } else if (lookup->input->stream_errno != 0) { - result->error = t_strdup_printf("read(%s) failed: %m", - lookup->path); - ret = -1; - } else if (lookup->input->eof) { - result->error = t_strdup_printf("Unexpected EOF from %s", - lookup->path); - ret = -1; + } else if (client->input->stream_errno != 0) { + dns_client_disconnect(client, t_strdup_printf( + "read(%s) failed: %s", client->path, + i_stream_get_error(client->input))); + return; + } else if (client->input->eof) { + dns_client_disconnect(client, t_strdup_printf( + "Unexpected EOF from %s", client->path)); + return; } - if (ret != 0) { + if (ret > 0) { dns_lookup_save_msecs(lookup); - dns_lookup_close(lookup); - lookup->callback(result, lookup->context); + lookup->callback(&lookup->result, lookup->context); + retry = !lookup->client->deinit_client_at_free; dns_lookup_free(&lookup); } + if (retry) + dns_client_input(client); } static void dns_lookup_timeout(struct dns_lookup *lookup) { lookup->result.error = "DNS lookup timed out"; - dns_lookup_close(lookup); lookup->callback(&lookup->result, lookup->context); dns_lookup_free(&lookup); } -static int -dns_lookup_common(const char *cmd, bool ptr_lookup, - const struct dns_lookup_settings *set, - dns_lookup_callback_t *callback, void *context, - struct dns_lookup **lookup_r) -{ - struct dns_lookup *lookup; - struct dns_lookup_result result; - int fd; - - memset(&result, 0, sizeof(result)); - result.ret = EAI_FAIL; - - fd = net_connect_unix(set->dns_client_socket_path); - if (fd == -1) { - result.error = t_strdup_printf("connect(%s) failed: %m", - set->dns_client_socket_path); - callback(&result, context); - return -1; - } - - if (write_full(fd, cmd, strlen(cmd)) < 0) { - result.error = t_strdup_printf("write(%s) failed: %m", - set->dns_client_socket_path); - i_close_fd(&fd); - callback(&result, context); - return -1; - } - - lookup = i_new(struct dns_lookup, 1); - lookup->ptr_lookup = ptr_lookup; - lookup->fd = fd; - lookup->path = i_strdup(set->dns_client_socket_path); - lookup->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); - lookup->io = io_add(fd, IO_READ, dns_lookup_input, lookup); - if (set->timeout_msecs != 0) { - lookup->to = timeout_add(set->timeout_msecs, - dns_lookup_timeout, lookup); - } - lookup->result.ret = EAI_FAIL; - lookup->callback = callback; - lookup->context = context; - if (gettimeofday(&lookup->start_time, NULL) < 0) - i_fatal("gettimeofday() failed: %m"); - - *lookup_r = lookup; - return 0; -} - -#undef dns_lookup int dns_lookup(const char *host, const struct dns_lookup_settings *set, dns_lookup_callback_t *callback, void *context, struct dns_lookup **lookup_r) { - return dns_lookup_common(t_strconcat("IP\t", host, "\n", NULL), FALSE, - set, callback, context, lookup_r); + struct dns_client *client; + + client = dns_client_init(set); + client->deinit_client_at_free = TRUE; + if (dns_client_lookup(client, host, callback, context, lookup_r) < 0) { + dns_client_deinit(&client); + return -1; + } + return 0; } -#undef dns_lookup_ptr int dns_lookup_ptr(const struct ip_addr *ip, const struct dns_lookup_settings *set, dns_lookup_callback_t *callback, void *context, struct dns_lookup **lookup_r) { - const char *cmd = t_strconcat("NAME\t", net_ip2addr(ip), "\n", NULL); - return dns_lookup_common(cmd, TRUE, set, callback, context, lookup_r); + struct dns_client *client; + + client = dns_client_init(set); + client->deinit_client_at_free = TRUE; + if (dns_client_lookup_ptr(client, ip, callback, context, lookup_r) < 0) { + dns_client_deinit(&client); + return -1; + } + return 0; +} + +static void dns_client_idle_timeout(struct dns_client *client) +{ + i_assert(client->head == NULL); + + dns_client_disconnect(client, "Idle timeout"); } static void dns_lookup_free(struct dns_lookup **_lookup) { struct dns_lookup *lookup = *_lookup; + struct dns_client *client = lookup->client; *_lookup = NULL; - dns_lookup_close(lookup); + DLLIST2_REMOVE(&client->head, &client->tail, lookup); + if (lookup->to != NULL) + timeout_remove(&lookup->to); i_free(lookup->name); i_free(lookup->ips); - i_free(lookup->path); + if (client->deinit_client_at_free) + dns_client_deinit(&client); + else if (client->head == NULL) { + client->to_idle = timeout_add(client->idle_timeout_msecs, + dns_client_idle_timeout, client); + } i_free(lookup); } @@ -242,5 +260,106 @@ { if (lookup->to != NULL) lookup->to = io_loop_move_timeout(&lookup->to); - lookup->io = io_loop_move_io(&lookup->io); + lookup->client->io = io_loop_move_io(&lookup->client->io); } + +struct dns_client *dns_client_init(const struct dns_lookup_settings *set) +{ + struct dns_client *client; + + client = i_new(struct dns_client, 1); + client->path = i_strdup(set->dns_client_socket_path); + client->timeout_msecs = set->timeout_msecs; + client->idle_timeout_msecs = set->idle_timeout_msecs; + client->fd = -1; From dovecot at dovecot.org Tue Oct 22 15:36:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Oct 2013 15:36:15 +0300 Subject: dovecot-2.2: lib-http: Support DNS lookups via the new dns-clien... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/864e74223d55 changeset: 16861:864e74223d55 user: Timo Sirainen date: Tue Oct 22 15:35:27 2013 +0300 description: lib-http: Support DNS lookups via the new dns-client API. diffstat: src/lib-http/http-client-host.c | 33 +++++++++++++++++---------------- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 5 +++++ 3 files changed, 23 insertions(+), 16 deletions(-) diffs (79 lines): diff -r a6abdf4b6222 -r 864e74223d55 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Tue Oct 22 15:34:55 2013 +0300 +++ b/src/lib-http/http-client-host.c Tue Oct 22 15:35:27 2013 +0300 @@ -394,18 +394,26 @@ unsigned int ips_count; int ret; - memset(&dns_set, 0, sizeof(dns_set)); - dns_set.dns_client_socket_path = - client->set.dns_client_socket_path; - dns_set.timeout_msecs = HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS; - - if (host->ips_count == 0 && - net_addr2ip(host->name, &ip) == 0) { // FIXME: remove this? + if (net_addr2ip(host->name, &ip) == 0) { host->ips_count = 1; host->ips = i_new(struct ip_addr, host->ips_count); host->ips[0] = ip; - } else if (dns_set.dns_client_socket_path == NULL) { - ret = net_gethostbyname(host->name, &ips, &ips_count); + } else if (client->set.dns_client != NULL) { + http_client_host_debug(host, + "Performing asynchronous DNS lookup"); + (void)dns_client_lookup(client->set.dns_client, host->name, + http_client_host_dns_callback, host, &host->dns_lookup); + } else if (dns_set.dns_client_socket_path != NULL) { + http_client_host_debug(host, + "Performing asynchronous DNS lookup"); + memset(&dns_set, 0, sizeof(dns_set)); + dns_set.dns_client_socket_path = + client->set.dns_client_socket_path; + dns_set.timeout_msecs = HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS; + (void)dns_lookup(host->name, &dns_set, + http_client_host_dns_callback, host, &host->dns_lookup); + } else { + ret = net_gethostbyname(host->name, &ips, &ips_count); if (ret != 0) { http_client_host_lookup_failure(host, net_gethosterror(ret)); return; @@ -418,13 +426,6 @@ host->ips = i_new(struct ip_addr, ips_count); memcpy(host->ips, ips, ips_count * sizeof(*ips)); } - - if (host->ips_count == 0) { - http_client_host_debug(host, - "Performing asynchronous DNS lookup"); - (void)dns_lookup(host->name, &dns_set, - http_client_host_dns_callback, host, &host->dns_lookup); - } } struct http_client_host *http_client_host_get diff -r a6abdf4b6222 -r 864e74223d55 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Tue Oct 22 15:34:55 2013 +0300 +++ b/src/lib-http/http-client.c Tue Oct 22 15:35:27 2013 +0300 @@ -75,6 +75,7 @@ pool = pool_alloconly_create("http client", 1024); client = p_new(pool, struct http_client, 1); client->pool = pool; + client->set.dns_client = set->dns_client; client->set.dns_client_socket_path = p_strdup_empty(pool, set->dns_client_socket_path); client->set.user_agent = p_strdup_empty(pool, set->user_agent); diff -r a6abdf4b6222 -r 864e74223d55 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Tue Oct 22 15:34:55 2013 +0300 +++ b/src/lib-http/http-client.h Tue Oct 22 15:35:27 2013 +0300 @@ -34,6 +34,11 @@ extern const char *http_request_state_names[]; struct http_client_settings { + /* a) If dns_client is set, all lookups are done via it. + b) If dns_client_socket_path is set, each DNS lookup does its own + dns-lookup UNIX socket connection. + c) Otherwise, blocking gethostbyname() lookups are used. */ + struct dns_client *dns_client; const char *dns_client_socket_path; const char *ssl_ca_dir, *ssl_ca_file, *ssl_ca; From dovecot at dovecot.org Tue Oct 22 15:36:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Oct 2013 15:36:15 +0300 Subject: dovecot-2.2: lib-fs: Added dns_client to fs_settings. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d23c8e97dc69 changeset: 16862:d23c8e97dc69 user: Timo Sirainen date: Tue Oct 22 15:36:07 2013 +0300 description: lib-fs: Added dns_client to fs_settings. diffstat: src/lib-fs/fs-api.h | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 864e74223d55 -r d23c8e97dc69 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Tue Oct 22 15:35:27 2013 +0300 +++ b/src/lib-fs/fs-api.h Tue Oct 22 15:36:07 2013 +0300 @@ -84,6 +84,9 @@ /* When creating temporary files, use this prefix (to avoid conflicts with existing files). */ const char *temp_file_prefix; + /* If the backend needs to do DNS lookups, use this dns_client for + them. */ + struct dns_client *dns_client; /* Enable debugging */ bool debug; From dovecot at dovecot.org Tue Oct 22 19:12:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 22 Oct 2013 19:12:49 +0300 Subject: dovecot-2.2: lib-dns: Fixed busy looping when dns-client disconn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/72b3b8c86222 changeset: 16863:72b3b8c86222 user: Timo Sirainen date: Tue Oct 22 19:12:38 2013 +0300 description: lib-dns: Fixed busy looping when dns-client disconnected. diffstat: src/lib-dns/dns-lookup.c | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diffs (15 lines): diff -r d23c8e97dc69 -r 72b3b8c86222 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Tue Oct 22 15:36:07 2013 +0300 +++ b/src/lib-dns/dns-lookup.c Tue Oct 22 19:12:38 2013 +0300 @@ -159,10 +159,8 @@ return; } } - if (ret == 0) - return; - if (lookup->result.error != NULL) { + if (ret != 0 && lookup->result.error != NULL) { /* already got the error */ } else if (client->input->stream_errno != 0) { dns_client_disconnect(client, t_strdup_printf( From dovecot at dovecot.org Wed Oct 23 11:38:21 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 11:38:21 +0300 Subject: dovecot-2.2: ioloop-kqueue: Added extra assert. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/10f3030047b6 changeset: 16864:10f3030047b6 user: Timo Sirainen date: Wed Oct 23 11:38:07 2013 +0300 description: ioloop-kqueue: Added extra assert. diffstat: src/lib/ioloop-kqueue.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 72b3b8c86222 -r 10f3030047b6 src/lib/ioloop-kqueue.c --- a/src/lib/ioloop-kqueue.c Tue Oct 22 19:12:38 2013 +0300 +++ b/src/lib/ioloop-kqueue.c Wed Oct 23 11:38:07 2013 +0300 @@ -85,6 +85,7 @@ struct ioloop_handler_context *ctx = io->io.ioloop->handler_context; struct kevent ev; + i_assert(io->io.condition != 0); if ((io->io.condition & (IO_READ | IO_ERROR)) != 0 && !closed) { MY_EV_SET(&ev, io->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); if (kevent(ctx->kq, &ev, 1, NULL, 0, NULL) < 0) @@ -95,6 +96,7 @@ if (kevent(ctx->kq, &ev, 1, NULL, 0, NULL) < 0) i_error("kevent(EV_DELETE, %d) failed: %m", io->fd); } + io->io.condition = 0; /* since we're not freeing memory in any case, just increase deleted counter so next handle_add() can just decrease it From dovecot at dovecot.org Wed Oct 23 11:40:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 11:40:34 +0300 Subject: dovecot-2.2: auth: Don't crash with "doveadm auth cache flush" w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c347c7faaf2 changeset: 16865:9c347c7faaf2 user: Timo Sirainen date: Wed Oct 23 11:40:26 2013 +0300 description: auth: Don't crash with "doveadm auth cache flush" when cache is disabled. diffstat: src/auth/auth-master-connection.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 10f3030047b6 -r 9c347c7faaf2 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Oct 23 11:38:07 2013 +0300 +++ b/src/auth/auth-master-connection.c Wed Oct 23 11:40:26 2013 +0300 @@ -147,7 +147,10 @@ return FALSE; } - if (list[1] == NULL) { + if (passdb_cache == NULL) { + /* cache disabled */ + count = 0; + } if (list[1] == NULL) { /* flush the whole cache */ count = auth_cache_clear(passdb_cache); } else { From dovecot at dovecot.org Wed Oct 23 11:47:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 11:47:15 +0300 Subject: dovecot-2.2: lib-auth: auth_master_cache_flush() always waited f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/100a2c0ce5e8 changeset: 16866:100a2c0ce5e8 user: Timo Sirainen date: Wed Oct 23 11:47:10 2013 +0300 description: lib-auth: auth_master_cache_flush() always waited for timeout before finishing. diffstat: src/lib-auth/auth-master.c | 25 +++++++++++++++++-------- 1 files changed, 17 insertions(+), 8 deletions(-) diffs (59 lines): diff -r 9c347c7faaf2 -r 100a2c0ce5e8 src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Wed Oct 23 11:40:26 2013 +0300 +++ b/src/lib-auth/auth-master.c Wed Oct 23 11:47:10 2013 +0300 @@ -588,30 +588,38 @@ return ctx.return_value; } +struct auth_master_cache_ctx { + struct auth_master_connection *conn; + unsigned int count; + bool failed; +}; + static bool auth_cache_flush_reply_callback(const char *cmd, const char *const *args, void *context) { - unsigned int *countp = context; + struct auth_master_cache_ctx *ctx = context; if (strcmp(cmd, "OK") != 0) - *countp = UINT_MAX; - else if (args[0] == NULL || str_to_uint(args[0], countp) < 0) - *countp = UINT_MAX; + ctx->failed = TRUE; + else if (args[0] == NULL || str_to_uint(args[0], &ctx->count) < 0) + ctx->failed = TRUE; - io_loop_stop(current_ioloop); + io_loop_stop(ctx->conn->ioloop); return TRUE; } int auth_master_cache_flush(struct auth_master_connection *conn, const char *const *users, unsigned int *count_r) { + struct auth_master_cache_ctx ctx; string_t *str; - *count_r = UINT_MAX; + memset(&ctx, 0, sizeof(ctx)); + ctx.conn = conn; conn->reply_callback = auth_cache_flush_reply_callback; - conn->reply_context = count_r; + conn->reply_context = &ctx; str = t_str_new(128); str_printfa(str, "CACHE-FLUSH\t%u", auth_master_next_request_id(conn)); @@ -628,7 +636,8 @@ conn->prefix = DEFAULT_USERDB_LOOKUP_PREFIX; conn->reply_context = NULL; - return *count_r == UINT_MAX ? -1 : 0; + *count_r = ctx.count; + return ctx.failed ? -1 : 0; } static bool From dovecot at dovecot.org Wed Oct 23 14:34:46 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 14:34:46 +0300 Subject: dovecot-2.2: doveadm-server: Don't call io_loop_run() recursivel... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d06acaf2c640 changeset: 16867:d06acaf2c640 user: Timo Sirainen date: Wed Oct 23 14:34:35 2013 +0300 description: doveadm-server: Don't call io_loop_run() recursively for the same ioloop. This breaks things more or less badly, especially ioloop-kqueue really didn't like it. diffstat: src/doveadm/client-connection.c | 18 ++++++++++++------ src/doveadm/doveadm-mail-server.c | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diffs (72 lines): diff -r 100a2c0ce5e8 -r d06acaf2c640 src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Wed Oct 23 11:47:10 2013 +0300 +++ b/src/doveadm/client-connection.c Wed Oct 23 14:34:35 2013 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2010-2013 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "lib-signals.h" #include "base64.h" #include "ioloop.h" #include "istream.h" @@ -107,10 +108,16 @@ const struct mail_storage_service_input *input) { const char *error; + struct ioloop *ioloop, *prev_ioloop = current_ioloop; int ret; ctx->conn = conn; + /* some commands will want to call io_loop_run(), but we're already + running one and we can't call the original one recursively, so + create a new ioloop. */ + ioloop = io_loop_create(); + lib_signals_reset_ioloop(); if (ctx->v.preinit != NULL) ctx->v.preinit(ctx); @@ -120,6 +127,11 @@ doveadm_print_flush(); mail_storage_service_deinit(&ctx->storage_service); + current_ioloop = prev_ioloop; + lib_signals_reset_ioloop(); + current_ioloop = ioloop; + io_loop_destroy(&ioloop); + if (ret < 0) { i_error("%s: %s", ctx->cmd->name, error); o_stream_nsend(conn->output, "\n-\n", 3); @@ -212,10 +224,6 @@ return FALSE; } - /* make sure client_connection_input() isn't called by the ioloop that - is going to be run by doveadm_mail_cmd_server_run() */ - io_remove(&conn->io); - o_stream_cork(conn->output); ctx = doveadm_mail_cmd_server_parse(cmd_name, conn->set, &input, argc, args); if (ctx == NULL) @@ -228,8 +236,6 @@ net_set_nonblock(conn->fd, FALSE); (void)o_stream_flush(conn->output); net_set_nonblock(conn->fd, TRUE); - - conn->io = io_add(conn->fd, IO_READ, client_connection_input, conn); return TRUE; } diff -r 100a2c0ce5e8 -r d06acaf2c640 src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Wed Oct 23 11:47:10 2013 +0300 +++ b/src/doveadm/doveadm-mail-server.c Wed Oct 23 14:34:35 2013 +0300 @@ -164,7 +164,7 @@ unsigned int count = array_count(&server->queue); do { - master_service_run(master_service, NULL); + io_loop_run(current_ioloop); } while (array_count(&server->queue) == count && doveadm_server_have_used_connections(server) && !DOVEADM_MAIL_SERVER_FAILED()); From dovecot at dovecot.org Wed Oct 23 14:35:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 14:35:12 +0300 Subject: dovecot-2.2: io_loop_run() now assert-crashes if it's attempted ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c82ac3b0474f changeset: 16868:c82ac3b0474f user: Timo Sirainen date: Wed Oct 23 14:35:07 2013 +0300 description: io_loop_run() now assert-crashes if it's attempted to be used recursively for the same ioloop. diffstat: src/lib/ioloop-private.h | 1 + src/lib/ioloop.c | 6 ++++++ 2 files changed, 7 insertions(+), 0 deletions(-) diffs (30 lines): diff -r d06acaf2c640 -r c82ac3b0474f src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Wed Oct 23 14:34:35 2013 +0300 +++ b/src/lib/ioloop-private.h Wed Oct 23 14:35:07 2013 +0300 @@ -25,6 +25,7 @@ time_t next_max_time; unsigned int running:1; + unsigned int iolooping:1; }; struct io { diff -r d06acaf2c640 -r c82ac3b0474f src/lib/ioloop.c --- a/src/lib/ioloop.c Wed Oct 23 14:34:35 2013 +0300 +++ b/src/lib/ioloop.c Wed Oct 23 14:35:07 2013 +0300 @@ -401,9 +401,15 @@ if (ioloop->cur_ctx != NULL) io_loop_context_unref(&ioloop->cur_ctx); + /* recursive io_loop_run() isn't allowed for the same ioloop. + it can break backends. */ + i_assert(!ioloop->iolooping); + ioloop->iolooping = TRUE; + ioloop->running = TRUE; while (ioloop->running) io_loop_handler_run(ioloop); + ioloop->iolooping = FALSE; } void io_loop_stop(struct ioloop *ioloop) From dovecot at dovecot.org Wed Oct 23 15:00:07 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 15:00:07 +0300 Subject: dovecot-2.2: doveadm-server: Fixed hangs caused by previous commit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2089c4b637b9 changeset: 16869:2089c4b637b9 user: Timo Sirainen date: Wed Oct 23 14:59:57 2013 +0300 description: doveadm-server: Fixed hangs caused by previous commit diffstat: src/doveadm/doveadm-mail-server.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r c82ac3b0474f -r 2089c4b637b9 src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Wed Oct 23 14:35:07 2013 +0300 +++ b/src/doveadm/doveadm-mail-server.c Wed Oct 23 14:59:57 2013 +0300 @@ -99,7 +99,7 @@ i_error("%s: Command %s failed for %s: %s", server->name, cmd_ctx->cmd->name, username, error); internal_failure = TRUE; - master_service_stop(master_service); + io_loop_stop(current_ioloop); return; case EX_NOUSER: i_error("%s: No such user: %s", server->name, username); @@ -125,7 +125,7 @@ } } - master_service_stop(master_service); + io_loop_stop(current_ioloop); } static void doveadm_mail_server_handle(struct server_connection *conn, From dovecot at dovecot.org Wed Oct 23 15:16:59 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 15:16:59 +0300 Subject: dovecot-2.2: dict-redis: Don't crash when receiving invalid inpu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a18a53f242ba changeset: 16870:a18a53f242ba user: Timo Sirainen date: Wed Oct 23 15:16:52 2013 +0300 description: dict-redis: Don't crash when receiving invalid input instead of expected $size. diffstat: src/lib-dict/dict-redis.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r 2089c4b637b9 -r a18a53f242ba src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Wed Oct 23 14:59:57 2013 +0300 +++ b/src/lib-dict/dict-redis.c Wed Oct 23 15:16:52 2013 +0300 @@ -137,8 +137,7 @@ if (line[0] != '$' || str_to_uint(line+1, &conn->bytes_left) < 0) { i_error("redis: Unexpected input (wanted $size): %s", line); - redis_conn_destroy(&conn->conn); - return 1; + return -1; } conn->bytes_left += 2; /* include trailing CRLF */ } From dovecot at dovecot.org Wed Oct 23 15:26:47 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 15:26:47 +0300 Subject: dovecot-2.2: dict-redis: Don't crash if we get disconnected duri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e721788d2946 changeset: 16871:e721788d2946 user: Timo Sirainen date: Wed Oct 23 15:26:35 2013 +0300 description: dict-redis: Don't crash if we get disconnected during an open transaction. diffstat: src/lib-dict/dict-redis.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r a18a53f242ba -r e721788d2946 src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Wed Oct 23 15:16:52 2013 +0300 +++ b/src/lib-dict/dict-redis.c Wed Oct 23 15:26:35 2013 +0300 @@ -591,6 +591,11 @@ if (ctx->failed) return -1; + if (!dict->connected) { + /* disconnected during transaction */ + ctx->failed = TRUE; + return -1; + } if (ctx->ctx.changed) return 0; From dovecot at dovecot.org Wed Oct 23 15:36:15 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 15:36:15 +0300 Subject: dovecot-2.2: replication plugin: Hide write(fifo) EPIPE errors, ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1dc6989b2c0 changeset: 16872:e1dc6989b2c0 user: Timo Sirainen date: Wed Oct 23 15:36:02 2013 +0300 description: replication plugin: Hide write(fifo) EPIPE errors, which just mean a server restart. diffstat: src/plugins/replication/replication-plugin.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (21 lines): diff -r e721788d2946 -r e1dc6989b2c0 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Wed Oct 23 15:26:35 2013 +0300 +++ b/src/plugins/replication/replication-plugin.c Wed Oct 23 15:36:02 2013 +0300 @@ -85,10 +85,14 @@ return 0; } if (ret != (ssize_t)str_len(str)) { - if (ret < 0) + if (ret > 0) + i_error("write(%s) wrote partial data", fifo_path); + else if (errno != EPIPE) i_error("write(%s) failed: %m", fifo_path); - else - i_error("write(%s) wrote partial data", fifo_path); + else { + /* server was probably restarted, don't bother logging + this. */ + } if (close(fifo_fd) < 0) i_error("close(%s) failed: %m", fifo_path); fifo_fd = -1; From dovecot at dovecot.org Wed Oct 23 16:15:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 16:15:52 +0300 Subject: dovecot-2.2: lib-master: If service_count=1, close the listener ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/459ec8f7ac96 changeset: 16873:459ec8f7ac96 user: Timo Sirainen date: Wed Oct 23 16:10:30 2013 +0300 description: lib-master: If service_count=1, close the listener before starting to handle the connection. This way if the connection handling takes a long time and the service doesn't notice that master dies, it can keep running without keeping the listener fds open and preventing a restart. diffstat: src/lib-master/master-service.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (27 lines): diff -r e1dc6989b2c0 -r 459ec8f7ac96 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Wed Oct 23 15:36:02 2013 +0300 +++ b/src/lib-master/master-service.c Wed Oct 23 16:10:30 2013 +0300 @@ -638,6 +638,15 @@ i_assert(service->master_status.available_count > 0); service->master_status.available_count--; master_status_update(service); + + if (service->master_status.available_count == 0 && + service->service_count_left == 1) { + /* we're not going to accept any more connections after this. + go ahead and close the connection early. */ + i_assert(service->listeners != NULL); + master_service_io_listeners_remove(service); + master_service_io_listeners_close(service); + } } void master_service_client_connection_accept(struct master_service_connection *conn) @@ -923,6 +932,7 @@ i_error("close(listener %d) failed: %m", service->listeners[i].fd); } + service->listeners[i].fd = -1; } } } else { From dovecot at dovecot.org Wed Oct 23 16:23:34 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 16:23:34 +0300 Subject: dovecot-2.2: stats: Hide warnings about old autocreated sessions... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25f6fbeff5d9 changeset: 16874:25f6fbeff5d9 user: Timo Sirainen date: Wed Oct 23 16:23:23 2013 +0300 description: stats: Hide warnings about old autocreated sessions having gotten lost. diffstat: src/stats/mail-session.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diffs (25 lines): diff -r 459ec8f7ac96 -r 25f6fbeff5d9 src/stats/mail-session.c --- a/src/stats/mail-session.c Wed Oct 23 16:10:30 2013 +0300 +++ b/src/stats/mail-session.c Wed Oct 23 16:23:23 2013 +0300 @@ -48,10 +48,17 @@ static void mail_session_idle_timeout(struct mail_session *session) { - i_warning("Session %s (user %s, service %s) appears to have crashed, " - "disconnecting it", - guid_128_to_string(session->guid), session->user->name, - session->service); + /* user="" service="" pid=0 is used for incoming sessions that were + received after we detected a stats process crash/restart. there's + no point in logging anything about them, since they contain no + useful information. */ + if (session->user->name[0] == '\0' && session->service[0] != '\0' && + session->pid == 0) { + i_warning("Session %s (user %s, service %s) " + "appears to have crashed, disconnecting it", + guid_128_to_string(session->guid), + session->user->name, session->service); + } mail_session_disconnect(session); } From dovecot at dovecot.org Wed Oct 23 16:33:51 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 16:33:51 +0300 Subject: dovecot-2.2: lib-master: Fix to previous commit. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/487afab10a4b changeset: 16875:487afab10a4b user: Timo Sirainen date: Wed Oct 23 16:33:43 2013 +0300 description: lib-master: Fix to previous commit. It broke doveadm-server's "does client need authentication?" check. diffstat: src/lib-master/master-service.c | 19 ++++++++++--------- 1 files changed, 10 insertions(+), 9 deletions(-) diffs (36 lines): diff -r 25f6fbeff5d9 -r 487afab10a4b src/lib-master/master-service.c --- a/src/lib-master/master-service.c Wed Oct 23 16:23:23 2013 +0300 +++ b/src/lib-master/master-service.c Wed Oct 23 16:33:43 2013 +0300 @@ -638,15 +638,6 @@ i_assert(service->master_status.available_count > 0); service->master_status.available_count--; master_status_update(service); - - if (service->master_status.available_count == 0 && - service->service_count_left == 1) { - /* we're not going to accept any more connections after this. - go ahead and close the connection early. */ - i_assert(service->listeners != NULL); - master_service_io_listeners_remove(service); - master_service_io_listeners_close(service); - } } void master_service_client_connection_accept(struct master_service_connection *conn) @@ -849,6 +840,16 @@ as real clients */ master_service_client_connection_destroyed(service); } + if (service->master_status.available_count == 0 && + service->service_count_left == 1) { + /* we're not going to accept any more connections after this. + go ahead and close the connection early. don't do this + before calling callback, because it may want to access + the listen_fd (e.g. to check socket permissions). */ + i_assert(service->listeners != NULL); + master_service_io_listeners_remove(service); + master_service_io_listeners_close(service); + } } static void io_listeners_init(struct master_service *service) From dovecot at dovecot.org Wed Oct 23 16:52:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 16:52:08 +0300 Subject: dovecot-2.2: doveadm expunge: Improved the error hint message ab... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ade042b2161d changeset: 16876:ade042b2161d user: Timo Sirainen date: Wed Oct 23 16:50:57 2013 +0300 description: doveadm expunge: Improved the error hint message about using something else besides MAILBOX. diffstat: src/doveadm/doveadm-mail-expunge.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 487afab10a4b -r ade042b2161d src/doveadm/doveadm-mail-expunge.c --- a/src/doveadm/doveadm-mail-expunge.c Wed Oct 23 16:33:43 2013 +0300 +++ b/src/doveadm/doveadm-mail-expunge.c Wed Oct 23 16:50:57 2013 +0300 @@ -224,7 +224,8 @@ if (!expunge_search_args_is_msgset_ok(args->args)) { i_fatal_status(EX_USAGE, "%s: To avoid accidents, each branch in search query " - "must contain something else besides MAILBOX", cmd); + "must contain something else besides MAILBOX " + "(e.g. just add \"all\" if you want everything)", cmd); } } From dovecot at dovecot.org Wed Oct 23 16:52:08 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 23 Oct 2013 16:52:08 +0300 Subject: dovecot-2.2: doveadm copy: Don't require the extra mailbox+extra... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/219f869cbd0d changeset: 16877:219f869cbd0d user: Timo Sirainen date: Wed Oct 23 16:51:55 2013 +0300 description: doveadm copy: Don't require the extra mailbox+extra parameter checks as expunge/move requires. Perhaps they wouldn't be necessary even with move. diffstat: src/doveadm/doveadm-mail-copymove.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r ade042b2161d -r 219f869cbd0d src/doveadm/doveadm-mail-copymove.c --- a/src/doveadm/doveadm-mail-copymove.c Wed Oct 23 16:50:57 2013 +0300 +++ b/src/doveadm/doveadm-mail-copymove.c Wed Oct 23 16:51:55 2013 +0300 @@ -152,7 +152,8 @@ ctx->destname = p_strdup(ctx->ctx.pool, destname); _ctx->search_args = doveadm_mail_build_search_args(args); - expunge_search_args_check(ctx->ctx.search_args, cmdname); + if (ctx->move) + expunge_search_args_check(ctx->ctx.search_args, cmdname); } static void cmd_copy_deinit(struct doveadm_mail_cmd_context *_ctx) From dovecot at dovecot.org Thu Oct 24 11:25:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Oct 2013 11:25:48 +0300 Subject: dovecot-2.2: doveadm mailbox create: Added -g parameter t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f89e645cba90 changeset: 16878:f89e645cba90 user: Timo Sirainen date: Thu Oct 24 11:25:41 2013 +0300 description: doveadm mailbox create: Added -g parameter to create mailbox with specific GUID. diffstat: src/doveadm/doveadm-mail-mailbox.c | 34 +++++++++++++++++++++++++++++----- 1 files changed, 29 insertions(+), 5 deletions(-) diffs (77 lines): diff -r 219f869cbd0d -r f89e645cba90 src/doveadm/doveadm-mail-mailbox.c --- a/src/doveadm/doveadm-mail-mailbox.c Wed Oct 23 16:51:55 2013 +0300 +++ b/src/doveadm/doveadm-mail-mailbox.c Thu Oct 24 11:25:41 2013 +0300 @@ -24,6 +24,12 @@ ARRAY_TYPE(const_string) mailboxes; }; +struct create_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + ARRAY_TYPE(const_string) mailboxes; + struct mailbox_update update; +}; + struct delete_cmd_context { struct doveadm_mailbox_cmd_context ctx; ARRAY_TYPE(const_string) mailboxes; @@ -197,7 +203,7 @@ cmd_mailbox_create_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) { - struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx; + struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx; struct mail_namespace *ns; struct mailbox *box; const char *const *namep; @@ -216,7 +222,7 @@ } box = mailbox_alloc(ns->list, name, 0); - if (mailbox_create(box, NULL, directory) < 0) { + if (mailbox_create(box, &ctx->update, directory) < 0) { i_error("Can't create mailbox %s: %s", name, mailbox_get_last_error(box, NULL)); doveadm_mail_failed_mailbox(_ctx, box); @@ -252,13 +258,31 @@ } } +static bool +cmd_mailbox_create_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx; + + switch (c) { + case 'g': + if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0) + doveadm_mail_help_name("mailbox create"); + break; + default: + return FALSE; + } + return TRUE; +} + static struct doveadm_mail_cmd_context *cmd_mailbox_create_alloc(void) { - struct mailbox_cmd_context *ctx; + struct create_cmd_context *ctx; - ctx = doveadm_mailbox_cmd_alloc(struct mailbox_cmd_context); + ctx = doveadm_mailbox_cmd_alloc(struct create_cmd_context); ctx->ctx.ctx.v.init = cmd_mailbox_create_init; ctx->ctx.ctx.v.run = cmd_mailbox_create_run; + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_create_parse_arg; + ctx->ctx.ctx.getopt_args = "g:"; p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); return &ctx->ctx.ctx; } @@ -532,7 +556,7 @@ }; struct doveadm_mail_cmd cmd_mailbox_create = { cmd_mailbox_create_alloc, "mailbox create", - "[-s] [...]" + "[-s] [-g ] [...]" }; struct doveadm_mail_cmd cmd_mailbox_delete = { cmd_mailbox_delete_alloc, "mailbox delete", From dovecot at dovecot.org Thu Oct 24 15:00:27 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Oct 2013 15:00:27 +0300 Subject: dovecot-2.2: auth: Cache master user logins also. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/082ca23fa9f4 changeset: 16879:082ca23fa9f4 user: Timo Sirainen date: Thu Oct 24 14:59:03 2013 +0300 description: auth: Cache master user logins also. diffstat: src/auth/auth-cache.c | 26 +++++++++++++++++--------- src/auth/auth-request.c | 12 +++++------- src/auth/auth-request.h | 2 +- src/auth/passdb-cache.c | 4 ++-- 4 files changed, 25 insertions(+), 19 deletions(-) diffs (139 lines): diff -r f89e645cba90 -r 082ca23fa9f4 src/auth/auth-cache.c --- a/src/auth/auth-cache.c Thu Oct 24 11:25:41 2013 +0300 +++ b/src/auth/auth-cache.c Thu Oct 24 14:59:03 2013 +0300 @@ -273,19 +273,24 @@ const char *data = node->data; unsigned int username_len; - /* The cache nodes begin with "P"/"U", passdb/userdb ID, "/" and - then usually followed by the username. It's too much trouble to - keep track of all the cache keys, so we'll just match it as if it - was the username. If e.g. '%n' is used in the cache key instead of - '%u', it means that cache entries can be removed only when @domain - isn't in the username parameter. */ + /* The cache nodes begin with "P"/"U", passdb/userdb ID, optional + "+" master user, "\t" and then usually followed by the username. + It's too much trouble to keep track of all the cache keys, so we'll + just match it as if it was the username. If e.g. '%n' is used in the + cache key instead of '%u', it means that cache entries can be + removed only when @domain isn't in the username parameter. */ if (*data != 'P' && *data != 'U') return FALSE; data++; while (*data >= '0' && *data <= '9') data++; - if (*data != '/') + if (*data == '+') { + /* skip over +master_user */ + while (*data != '\t' && *data != '\0') + data++; + } + if (*data != '\t') return FALSE; data++; @@ -339,7 +344,9 @@ /* Uniquely identify the request's passdb/userdb with the P/U prefix and by "%!", which expands to the passdb/userdb ID number. */ - key = t_strconcat(request->userdb_lookup ? "U" : "P", "%!/", key, NULL); + key = t_strconcat(request->userdb_lookup ? "U" : "P", "%!", + request->master_user == NULL ? "" : "+%{master_user}", + "\t", key, NULL); str = t_str_new(256); var_expand(str, key, @@ -407,7 +414,8 @@ a master user login */ current_username = request->user; if (request->translated_username != NULL && - request->requested_login_user == NULL) + request->requested_login_user == NULL && + request->master_user == NULL) request->user = t_strdup_noconst(request->translated_username); key = auth_request_expand_cache_key(request, key); diff -r f89e645cba90 -r 082ca23fa9f4 src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Oct 24 11:25:41 2013 +0300 +++ b/src/auth/auth-request.c Thu Oct 24 14:59:03 2013 +0300 @@ -409,8 +409,7 @@ i_unreached(); } - if (passdb_cache == NULL || passdb->cache_key == NULL || - request->master_user != NULL) + if (passdb_cache == NULL || passdb->cache_key == NULL) return; if (result < 0) { @@ -923,8 +922,7 @@ string_t *str; const char *cache_value; - if (passdb_cache == NULL || userdb->cache_key == NULL || - request->master_user != NULL) + if (passdb_cache == NULL || userdb->cache_key == NULL) return; if (result == USERDB_RESULT_USER_UNKNOWN) @@ -956,9 +954,6 @@ struct auth_cache_node *node; bool expired, neg_expired; - if (request->master_user != NULL) - return FALSE; - value = auth_cache_lookup(passdb_cache, request, key, &node, &expired, &neg_expired); if (value == NULL || (expired && !use_expired)) { @@ -1951,6 +1946,7 @@ { '\0', NULL, "real_rport" }, { '\0', NULL, "domain_first" }, { '\0', NULL, "domain_last" }, + { '\0', NULL, "master_user" }, /* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */ { '\0', NULL, NULL } }; @@ -2036,6 +2032,8 @@ tab[24].value = strrchr(auth_request->user, '@'); if (tab[24].value != NULL) tab[24].value = escape_func(tab[24].value+1, auth_request); + tab[25].value = auth_request->master_user == NULL ? NULL : + escape_func(auth_request->master_user, auth_request); return ret_tab; } diff -r f89e645cba90 -r 082ca23fa9f4 src/auth/auth-request.h --- a/src/auth/auth-request.h Thu Oct 24 11:25:41 2013 +0300 +++ b/src/auth/auth-request.h Thu Oct 24 14:59:03 2013 +0300 @@ -143,7 +143,7 @@ #define AUTH_REQUEST_VAR_TAB_USER_IDX 0 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2 -#define AUTH_REQUEST_VAR_TAB_COUNT 25 +#define AUTH_REQUEST_VAR_TAB_COUNT 26 extern const struct var_expand_table auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1]; diff -r f89e645cba90 -r 082ca23fa9f4 src/auth/passdb-cache.c --- a/src/auth/passdb-cache.c Thu Oct 24 11:25:41 2013 +0300 +++ b/src/auth/passdb-cache.c Thu Oct 24 14:59:03 2013 +0300 @@ -33,7 +33,7 @@ int ret; bool expired, neg_expired; - if (passdb_cache == NULL || key == NULL || request->master_user != NULL) + if (passdb_cache == NULL || key == NULL) return FALSE; /* value = password \t ... */ @@ -97,7 +97,7 @@ struct auth_cache_node *node; bool expired, neg_expired; - if (passdb_cache == NULL || request->master_user != NULL) + if (passdb_cache == NULL) return FALSE; value = auth_cache_lookup(passdb_cache, request, key, &node, From dovecot at dovecot.org Thu Oct 24 16:08:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Oct 2013 16:08:29 +0300 Subject: dovecot-2.2: login proxy: Use corking when writing data. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2d3d73d03fe2 changeset: 16880:2d3d73d03fe2 user: Timo Sirainen date: Thu Oct 24 16:08:23 2013 +0300 description: login proxy: Use corking when writing data. diffstat: src/login-common/client-common-auth.c | 6 ++++++ src/login-common/login-proxy.c | 22 ++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diffs (82 lines): diff -r 082ca23fa9f4 -r 2d3d73d03fe2 src/login-common/client-common-auth.c --- a/src/login-common/client-common-auth.c Thu Oct 24 14:59:03 2013 +0300 +++ b/src/login-common/client-common-auth.c Thu Oct 24 16:08:23 2013 +0300 @@ -222,6 +222,7 @@ static void proxy_input(struct client *client) { struct istream *input; + struct ostream *output; const char *line; unsigned int duration; @@ -265,10 +266,15 @@ return; } + output = client->output; + o_stream_ref(output); + o_stream_cork(output); while ((line = i_stream_next_line(input)) != NULL) { if (client->v.proxy_parse_line(client, line) != 0) break; } + o_stream_uncork(output); + o_stream_unref(&output); } static int proxy_start(struct client *client, diff -r 082ca23fa9f4 -r 2d3d73d03fe2 src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Thu Oct 24 14:59:03 2013 +0300 +++ b/src/login-common/login-proxy.c Thu Oct 24 16:08:23 2013 +0300 @@ -78,7 +78,7 @@ static void server_input(struct login_proxy *proxy) { unsigned char buf[OUTBUF_THRESHOLD]; - ssize_t ret; + ssize_t ret, ret2; proxy->last_io = ioloop_time; if (o_stream_get_buffer_used_size(proxy->client_output) > @@ -90,9 +90,14 @@ } ret = net_receive(proxy->server_fd, buf, sizeof(buf)); - if (ret < 0) + if (ret < 0) { login_proxy_free_errno(&proxy, errno, "server"); - else if (o_stream_send(proxy->client_output, buf, ret) != ret) { + return; + } + o_stream_cork(proxy->client_output); + ret2 = o_stream_send(proxy->client_output, buf, ret); + o_stream_uncork(proxy->client_output); + if (ret2 != ret) { login_proxy_free_errno(&proxy, proxy->client_output->stream_errno, "client"); @@ -102,7 +107,7 @@ static void proxy_client_input(struct login_proxy *proxy) { unsigned char buf[OUTBUF_THRESHOLD]; - ssize_t ret; + ssize_t ret, ret2; proxy->last_io = ioloop_time; if (o_stream_get_buffer_used_size(proxy->server_output) > @@ -114,9 +119,14 @@ } ret = net_receive(proxy->client_fd, buf, sizeof(buf)); - if (ret < 0) + if (ret < 0) { login_proxy_free_errno(&proxy, errno, "client"); - else if (o_stream_send(proxy->server_output, buf, ret) != ret) { + return; + } + o_stream_cork(proxy->client_output); + ret2 = o_stream_send(proxy->server_output, buf, ret); + o_stream_uncork(proxy->server_output); + if (ret2 != ret) { login_proxy_free_errno(&proxy, proxy->server_output->stream_errno, "server"); From dovecot at dovecot.org Thu Oct 24 16:21:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 24 Oct 2013 16:21:20 +0300 Subject: dovecot-2.2: *-login: Send the auth reply back corked. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5bb04fe417b changeset: 16881:e5bb04fe417b user: Timo Sirainen date: Thu Oct 24 16:21:10 2013 +0300 description: *-login: Send the auth reply back corked. diffstat: src/login-common/client-common-auth.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 2d3d73d03fe2 -r e5bb04fe417b src/login-common/client-common-auth.c --- a/src/login-common/client-common-auth.c Thu Oct 24 16:08:23 2013 +0300 +++ b/src/login-common/client-common-auth.c Thu Oct 24 16:21:10 2013 +0300 @@ -361,7 +361,9 @@ client_auth_result(struct client *client, enum client_auth_result result, const struct client_auth_reply *reply, const char *text) { + o_stream_cork(client->output); client->v.auth_result(client, result, reply, text); + o_stream_uncork(client->output); } static bool From dovecot at dovecot.org Sat Oct 26 18:00:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 18:00:29 +0300 Subject: dovecot-2.1: imap: Fixed sending LIST-STATUS for selectable name... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/d16e212531ec changeset: 15005:d16e212531ec user: Timo Sirainen date: Sat Oct 26 18:00:07 2013 +0300 description: imap: Fixed sending LIST-STATUS for selectable namespace prefixes. Most importantly sending it for INBOX when namespace prefix was also INBOX. diffstat: src/imap/cmd-list.c | 65 +++++++++++++++++++++++++++------------------------- 1 files changed, 34 insertions(+), 31 deletions(-) diffs (83 lines): diff -r 9313b0733008 -r d16e212531ec src/imap/cmd-list.c --- a/src/imap/cmd-list.c Wed Oct 02 08:02:30 2013 +0300 +++ b/src/imap/cmd-list.c Sat Oct 26 18:00:07 2013 +0300 @@ -344,6 +344,38 @@ } static void +list_send_status(struct cmd_list_context *ctx, const char *name, + const char *mutf7_name, enum mailbox_info_flags flags) +{ + struct imap_status_result result; + struct mail_namespace *ns; + const char *error; + + if ((flags & (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0) { + /* doesn't exist, don't even try to get STATUS */ + return; + } + if ((flags & MAILBOX_SUBSCRIBED) == 0 && + (flags & MAILBOX_CHILD_SUBSCRIBED) != 0) { + /* listing subscriptions, but only child is subscribed */ + return; + } + + /* if we're listing subscriptions and there are subscriptions=no + namespaces, ctx->ns may not point to correct one */ + ns = mail_namespace_find(ctx->ns->user->namespaces, name); + if (imap_status_get(ctx->cmd, ns, name, + &ctx->status_items, &result, &error) < 0) { + client_send_line(ctx->cmd->client, + t_strconcat("* ", error, NULL)); + return; + } + + imap_status_send(ctx->cmd->client, mutf7_name, + &ctx->status_items, &result); +} + +static void list_namespace_send_prefix(struct cmd_list_context *ctx, bool have_children) { struct mail_namespace *const *listed; @@ -440,38 +472,9 @@ mailbox_childinfo2str(ctx, str, flags); client_send_line(ctx->cmd->client, str_c(str)); -} -static void -list_send_status(struct cmd_list_context *ctx, const char *name, - const char *mutf7_name, enum mailbox_info_flags flags) -{ - struct imap_status_result result; - struct mail_namespace *ns; - const char *error; - - if ((flags & (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0) { - /* doesn't exist, don't even try to get STATUS */ - return; - } - if ((flags & MAILBOX_SUBSCRIBED) == 0 && - (flags & MAILBOX_CHILD_SUBSCRIBED) != 0) { - /* listing subscriptions, but only child is subscribed */ - return; - } - - /* if we're listing subscriptions and there are subscriptions=no - namespaces, ctx->ns may not point to correct one */ - ns = mail_namespace_find(ctx->ns->user->namespaces, name); - if (imap_status_get(ctx->cmd, ns, name, - &ctx->status_items, &result, &error) < 0) { - client_send_line(ctx->cmd->client, - t_strconcat("* ", error, NULL)); - return; - } - - imap_status_send(ctx->cmd->client, mutf7_name, - &ctx->status_items, &result); + if (ctx->used_status) + list_send_status(ctx, name, str_c(mutf7_name), flags); } static bool list_has_empty_prefix_ns(struct mail_user *user) From dovecot at dovecot.org Sat Oct 26 18:07:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 18:07:52 +0300 Subject: dovecot-2.2: auth: Fixed assert-crash with auth_verbose_password... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a8e63a351f5 changeset: 16882:8a8e63a351f5 user: Timo Sirainen date: Sat Oct 26 18:06:26 2013 +0300 description: auth: Fixed assert-crash with auth_verbose_passwords!=no and 32bit systems. t_strndup() assert-crashed with size=UINT_MAX parameter. diffstat: src/auth/auth-request.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e5bb04fe417b -r 8a8e63a351f5 src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Oct 24 16:21:10 2013 +0300 +++ b/src/auth/auth-request.c Sat Oct 26 18:06:26 2013 +0300 @@ -1781,7 +1781,7 @@ auth_request_append_password(struct auth_request *request, string_t *str) { const char *p, *log_type = request->set->verbose_passwords; - unsigned int max_len = UINT_MAX; + unsigned int max_len = 1024; p = strchr(log_type, ':'); if (p != NULL) { From dovecot at dovecot.org Sat Oct 26 18:07:52 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 18:07:52 +0300 Subject: dovecot-2.2: auth: Fixed crash with auth_verbose_passwords!=no a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d222bd0a49c changeset: 16883:1d222bd0a49c user: Timo Sirainen date: Sat Oct 26 18:07:45 2013 +0300 description: auth: Fixed crash with auth_verbose_passwords!=no and non-plaintext auth diffstat: src/auth/auth-request.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 8a8e63a351f5 -r 1d222bd0a49c src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Oct 26 18:06:26 2013 +0300 +++ b/src/auth/auth-request.c Sat Oct 26 18:07:45 2013 +0300 @@ -1783,6 +1783,9 @@ const char *p, *log_type = request->set->verbose_passwords; unsigned int max_len = 1024; + if (request->mech_password == NULL) + return; + p = strchr(log_type, ':'); if (p != NULL) { if (str_to_uint(p+1, &max_len) < 0) From dovecot at dovecot.org Sat Oct 26 18:31:29 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 18:31:29 +0300 Subject: dovecot-2.2: dsync: Fixed crashes at deinit when -r rawlog param... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d66b4b1b343a changeset: 16884:d66b4b1b343a user: Timo Sirainen date: Sat Oct 26 18:31:16 2013 +0300 description: dsync: Fixed crashes at deinit when -r rawlog parameter was used. diffstat: src/doveadm/dsync/doveadm-dsync.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 1d222bd0a49c -r d66b4b1b343a src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sat Oct 26 18:07:45 2013 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Sat Oct 26 18:31:16 2013 +0300 @@ -455,12 +455,12 @@ ctx->input = i_stream_create_fd(ctx->fd_in, (size_t)-1, FALSE); ctx->output = o_stream_create_fd(ctx->fd_out, (size_t)-1, FALSE); } - i_stream_ref(ctx->input); - o_stream_ref(ctx->output); if (ctx->rawlog_path != NULL) { iostream_rawlog_create_path(ctx->rawlog_path, &ctx->input, &ctx->output); } + i_stream_ref(ctx->input); + o_stream_ref(ctx->output); return dsync_ibc_init_stream(ctx->input, ctx->output, name, temp_prefix); } From dovecot at dovecot.org Sat Oct 26 19:00:03 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 19:00:03 +0300 Subject: dovecot-2.2: lib-http: Recent DNS change caused lib-http to acce... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/519b15c7618a changeset: 16885:519b15c7618a user: Timo Sirainen date: Sat Oct 26 18:59:03 2013 +0300 description: lib-http: Recent DNS change caused lib-http to access uninitialized memory. diffstat: src/lib-http/http-client-host.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d66b4b1b343a -r 519b15c7618a src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Oct 26 18:31:16 2013 +0300 +++ b/src/lib-http/http-client-host.c Sat Oct 26 18:59:03 2013 +0300 @@ -403,7 +403,7 @@ "Performing asynchronous DNS lookup"); (void)dns_client_lookup(client->set.dns_client, host->name, http_client_host_dns_callback, host, &host->dns_lookup); - } else if (dns_set.dns_client_socket_path != NULL) { + } else if (client->set.dns_client_socket_path != NULL) { http_client_host_debug(host, "Performing asynchronous DNS lookup"); memset(&dns_set, 0, sizeof(dns_set)); From dovecot at dovecot.org Sat Oct 26 19:00:03 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 19:00:03 +0300 Subject: dovecot-2.2: fts-solr: Fixed support for multiple Solr hosts whe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26355654c314 changeset: 16886:26355654c314 user: Timo Sirainen date: Sat Oct 26 18:59:52 2013 +0300 description: fts-solr: Fixed support for multiple Solr hosts when same process handles multiple users. E.g. indexer-worker was always using only the first user's Solr host. diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 29 +++++++++++++++--------- src/plugins/fts-solr/fts-backend-solr.c | 34 ++++++++++++++++++---------- src/plugins/fts-solr/fts-solr-plugin.c | 7 +++-- src/plugins/fts-solr/fts-solr-plugin.h | 3 +- src/plugins/fts-solr/solr-connection.c | 32 ++++++++++++-------------- 5 files changed, 61 insertions(+), 44 deletions(-) diffs (truncated from 379 to 300 lines): diff -r 519b15c7618a -r 26355654c314 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Sat Oct 26 18:59:03 2013 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Sat Oct 26 18:59:52 2013 +0300 @@ -22,6 +22,7 @@ struct solr_fts_backend { struct fts_backend backend; + struct solr_connection *solr_conn; char *id_username, *id_namespace; struct mail_namespace *default_ns; }; @@ -235,11 +236,9 @@ *error_r = "Invalid fts_solr setting"; return -1; } - if (solr_conn == NULL) { - if (solr_connection_init(fuser->set.url, fuser->set.debug, - &solr_conn, error_r) < 0) - return -1; - } + if (solr_connection_init(fuser->set.url, fuser->set.debug, + &backend->solr_conn, error_r) < 0) + return -1; str = solr_escape_id_str(_backend->ns->user->username); backend->id_username = i_strdup(str); @@ -315,7 +314,7 @@ solr_quote_http(str, ns->user->username); pool = pool_alloconly_create("solr last uid lookup", 1024); - if (solr_connection_select(solr_conn, str_c(str), + if (solr_connection_select(backend->solr_conn, str_c(str), pool, &results) < 0) ret = -1; else if (results[0] == NULL) { @@ -442,6 +441,8 @@ { struct solr_fts_backend_update_context *ctx = (struct solr_fts_backend_update_context *)_ctx; + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_ctx->backend; const char *str; int ret; @@ -452,7 +453,7 @@ str = t_strdup_printf("", ctx->documents_added ? "true" : "false"); - if (solr_connection_post(solr_conn, str) < 0) + if (solr_connection_post(backend->solr_conn, str) < 0) ret = -1; str_free(&ctx->cmd); @@ -494,6 +495,8 @@ { struct solr_fts_backend_update_context *ctx = (struct solr_fts_backend_update_context *)_ctx; + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_ctx->backend; T_BEGIN { string_t *cmd; @@ -503,7 +506,7 @@ xml_encode_id(ctx, cmd, uid); str_append(cmd, ""); - (void)solr_connection_post(solr_conn, str_c(cmd)); + (void)solr_connection_post(backend->solr_conn, str_c(cmd)); } T_END; } @@ -511,10 +514,13 @@ fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx, uint32_t uid) { + struct solr_fts_backend *backend = + (struct solr_fts_backend *)ctx->ctx.backend; + if (ctx->post == NULL) { i_assert(ctx->prev_uid == 0); - ctx->post = solr_connection_post_begin(solr_conn); + ctx->post = solr_connection_post_begin(backend->solr_conn); str_append(ctx->cmd, ""); } else { ctx->headers_open = FALSE; @@ -695,7 +701,8 @@ solr_add_ns_query_http(str, backend, ns); pool = pool_alloconly_create("fts solr search", 1024); - ret = solr_connection_select(solr_conn, str_c(str), pool, &results); + ret = solr_connection_select(backend->solr_conn, str_c(str), + pool, &results); if (ret == 0 && results[0] != NULL) { array_append_array(&result->definite_uids, &results[0]->uids); array_append_array(&result->scores, &results[0]->scores); @@ -761,7 +768,7 @@ } str_append_c(str, ')'); - if (solr_connection_select(solr_conn, str_c(str), + if (solr_connection_select(backend->solr_conn, str_c(str), result->pool, &solr_results) < 0) { hash_table_destroy(&mailboxes); return -1; diff -r 519b15c7618a -r 26355654c314 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Sat Oct 26 18:59:03 2013 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Sat Oct 26 18:59:52 2013 +0300 @@ -28,6 +28,7 @@ struct solr_fts_backend { struct fts_backend backend; + struct solr_connection *solr_conn; }; struct solr_fts_field { @@ -157,18 +158,15 @@ static int fts_backend_solr_init(struct fts_backend *_backend, const char **error_r) { + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user); if (fuser == NULL) { *error_r = "Invalid fts_solr setting"; return -1; } - if (solr_conn == NULL) { - if (solr_connection_init(fuser->set.url, fuser->set.debug, - &solr_conn, error_r) < 0) - return -1; - } - return 0; + return solr_connection_init(fuser->set.url, fuser->set.debug, + &backend->solr_conn, error_r); } static void fts_backend_solr_deinit(struct fts_backend *_backend) @@ -182,6 +180,7 @@ get_last_uid_fallback(struct fts_backend *_backend, struct mailbox *box, uint32_t *last_uid_r) { + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; const struct seq_range *uidvals; const char *box_guid; unsigned int count; @@ -203,7 +202,7 @@ str_append(str, "%22%22"); pool = pool_alloconly_create("solr last uid lookup", 1024); - if (solr_connection_select(solr_conn, str_c(str), + if (solr_connection_select(backend->solr_conn, str_c(str), pool, &results) < 0) ret = -1; else if (results[0] == NULL) { @@ -338,8 +337,11 @@ static void fts_backend_solr_expunge_flush(struct solr_fts_backend_update_context *ctx) { + struct solr_fts_backend *backend = + (struct solr_fts_backend *)ctx->ctx.backend; + str_append(ctx->cmd_expunge, ""); - (void)solr_connection_post(solr_conn, str_c(ctx->cmd_expunge)); + (void)solr_connection_post(backend->solr_conn, str_c(ctx->cmd_expunge)); str_truncate(ctx->cmd_expunge, 0); str_append(ctx->cmd_expunge, ""); } @@ -349,6 +351,8 @@ { struct solr_fts_backend_update_context *ctx = (struct solr_fts_backend_update_context *)_ctx; + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_ctx->backend; struct solr_fts_field *field; const char *str; int ret = _ctx->failed ? -1 : 0; @@ -363,7 +367,7 @@ fts_backend_solr_expunge_flush(ctx); str = t_strdup_printf("", ctx->documents_added ? "true" : "false"); - if (solr_connection_post(solr_conn, str) < 0) + if (solr_connection_post(backend->solr_conn, str) < 0) ret = -1; } @@ -444,11 +448,14 @@ fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx, uint32_t uid) { + struct solr_fts_backend *backend = + (struct solr_fts_backend *)ctx->ctx.backend; + if (ctx->post == NULL) { i_assert(ctx->prev_uid == 0); ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); - ctx->post = solr_connection_post_begin(solr_conn); + ctx->post = solr_connection_post_begin(backend->solr_conn); str_append(ctx->cmd, ""); } else { fts_backend_solr_doc_close(ctx); @@ -751,6 +758,7 @@ const char *box_guid, ARRAY_TYPE(seq_range) *uids_r, ARRAY_TYPE(fts_score_map) *scores_r) { + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; pool_t pool = pool_alloconly_create("fts solr search", 1024); struct solr_result **results; int ret; @@ -763,7 +771,8 @@ else str_append(str, "%22%22"); - ret = solr_connection_select(solr_conn, str_c(str), pool, &results); + ret = solr_connection_select(backend->solr_conn, str_c(str), + pool, &results); if (ret == 0 && results[0] != NULL) { array_append_array(uids_r, &results[0]->uids); array_append_array(scores_r, &results[0]->scores); @@ -811,6 +820,7 @@ struct mailbox *const boxes[], struct fts_multi_result *result) { + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; struct solr_result **solr_results; struct fts_result *fts_result; ARRAY(struct fts_result) fts_results; @@ -842,7 +852,7 @@ } str_append_c(str, ')'); - if (solr_connection_select(solr_conn, str_c(str), + if (solr_connection_select(backend->solr_conn, str_c(str), result->pool, &solr_results) < 0) { hash_table_destroy(&mailboxes); return -1; diff -r 519b15c7618a -r 26355654c314 src/plugins/fts-solr/fts-solr-plugin.c --- a/src/plugins/fts-solr/fts-solr-plugin.c Sat Oct 26 18:59:03 2013 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Sat Oct 26 18:59:52 2013 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "http-client.h" #include "mail-user.h" #include "mail-storage-hooks.h" #include "solr-connection.h" @@ -10,7 +11,7 @@ #include const char *fts_solr_plugin_version = DOVECOT_ABI_VERSION; -struct solr_connection *solr_conn = NULL; +struct http_client *solr_http_client = NULL; struct fts_solr_user_module fts_solr_user_module = MODULE_CONTEXT_INIT(&mail_user_module_register); @@ -84,8 +85,8 @@ fts_backend_unregister(fts_backend_solr.name); fts_backend_unregister(fts_backend_solr_old.name); mail_storage_hooks_remove(&fts_solr_mail_storage_hooks); - if (solr_conn != NULL) - solr_connection_deinit(solr_conn); + if (solr_http_client != NULL) + http_client_deinit(&solr_http_client); } diff -r 519b15c7618a -r 26355654c314 src/plugins/fts-solr/fts-solr-plugin.h --- a/src/plugins/fts-solr/fts-solr-plugin.h Sat Oct 26 18:59:03 2013 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.h Sat Oct 26 18:59:52 2013 +0300 @@ -2,6 +2,7 @@ #define FTS_SOLR_PLUGIN_H #include "module-context.h" +#include "mail-user.h" #include "fts-api-private.h" #define FTS_SOLR_USER_CONTEXT(obj) \ @@ -21,7 +22,7 @@ extern struct fts_backend fts_backend_solr; extern struct fts_backend fts_backend_solr_old; extern MODULE_CONTEXT_DEFINE(fts_solr_user_module, &mail_user_module_register); -extern struct solr_connection *solr_conn; +extern struct http_client *solr_http_client; void fts_solr_plugin_init(struct module *module); void fts_solr_plugin_deinit(void); diff -r 519b15c7618a -r 26355654c314 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Sat Oct 26 18:59:03 2013 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Sat Oct 26 18:59:52 2013 +0300 @@ -9,6 +9,7 @@ #include "istream.h" #include "http-url.h" #include "http-client.h" +#include "fts-solr-plugin.h" #include "solr-connection.h" #include @@ -54,8 +55,6 @@ From dovecot at dovecot.org Sat Oct 26 19:11:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 26 Oct 2013 19:11:48 +0300 Subject: dovecot-2.2: auth: Refuse to run checkpassword script insecurely... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a13098b642e9 changeset: 16887:a13098b642e9 user: Timo Sirainen date: Sat Oct 26 19:11:34 2013 +0300 description: auth: Refuse to run checkpassword script insecurely by default diffstat: src/auth/checkpassword-reply.c | 27 ++++++++++++++++++++++++++- src/auth/db-checkpassword.c | 1 + 2 files changed, 27 insertions(+), 1 deletions(-) diffs (54 lines): diff -r 26355654c314 -r a13098b642e9 src/auth/checkpassword-reply.c --- a/src/auth/checkpassword-reply.c Sat Oct 26 18:59:52 2013 +0300 +++ b/src/auth/checkpassword-reply.c Sat Oct 26 19:11:34 2013 +0300 @@ -11,13 +11,38 @@ int main(void) { string_t *str; - const char *user, *home, *authorized; + const char *user, *home, *authorized, *orig_uid; const char *extra_env, *key, *value, *const *tmp; bool uid_found = FALSE, gid_found = FALSE; lib_init(); str = t_str_new(1024); + orig_uid = getenv("ORIG_UID"); + /* ORIG_UID should have the auth process's UID that forked us. + if the checkpassword changed the UID, this could be a security hole + because the UID's other processes can ptrace this process and write + any kind of a reply to fd 4. so we can run only if: + + a) INSECURE_SETUID environment is set. + b) process isn't ptraceable (this binary is setuid/setgid) + c) checkpassword didn't actually change the UID (but used + userdb_uid instead) + */ + if (getenv("INSECURE_SETUID") == NULL && + (orig_uid == NULL || strtoul(orig_uid, NULL, 10) != getuid()) && + getuid() == geteuid() && getgid() == getegid()) { + if (orig_uid == NULL) { + i_error("checkpassword: ORIG_UID environment was dropped by checkpassword. " + "Can't verify if we're safe to run. See " + "http://wiki2.dovecot.org/AuthDatabase/CheckPassword#Security"); + } else { + i_error("checkpassword: The checkpassword couldn't be run securely. See " + "http://wiki2.dovecot.org/AuthDatabase/CheckPassword#Security"); + } + return 111; + } + user = getenv("USER"); if (user != NULL) { if (strchr(user, '\t') != NULL) { diff -r 26355654c314 -r a13098b642e9 src/auth/db-checkpassword.c --- a/src/auth/db-checkpassword.c Sat Oct 26 18:59:52 2013 +0300 +++ b/src/auth/db-checkpassword.c Sat Oct 26 19:11:34 2013 +0300 @@ -251,6 +251,7 @@ pipe, also pass some other possibly interesting information via environment. Use UCSPI names for local/remote IPs. */ env_put("PROTO=TCP"); /* UCSPI */ + env_put(t_strdup_printf("ORIG_UID=%s", dec2str(getuid()))); env_put(t_strconcat("SERVICE=", request->service, NULL)); if (request->local_ip.family != 0) { env_put(t_strconcat("TCPLOCALIP=", From dovecot at dovecot.org Mon Oct 28 11:50:49 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Oct 2013 11:50:49 +0200 Subject: dovecot-2.2: doveadm-server: ioloop change caused assert-crashes... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac59c37ae57b changeset: 16888:ac59c37ae57b user: Timo Sirainen date: Mon Oct 28 11:50:14 2013 +0200 description: doveadm-server: ioloop change caused assert-crashes if outgoing data was buffered at the end. diffstat: src/doveadm/client-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a13098b642e9 -r ac59c37ae57b src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Sat Oct 26 19:11:34 2013 +0300 +++ b/src/doveadm/client-connection.c Mon Oct 28 11:50:14 2013 +0200 @@ -129,6 +129,7 @@ current_ioloop = prev_ioloop; lib_signals_reset_ioloop(); + o_stream_switch_ioloop(conn->output); current_ioloop = ioloop; io_loop_destroy(&ioloop); From dovecot at dovecot.org Mon Oct 28 11:50:50 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 28 Oct 2013 11:50:50 +0200 Subject: dovecot-2.2: doveadm-server: Minor error message improvement. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2599056381cc changeset: 16889:2599056381cc user: Timo Sirainen date: Mon Oct 28 11:50:40 2013 +0200 description: doveadm-server: Minor error message improvement. diffstat: src/doveadm/client-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ac59c37ae57b -r 2599056381cc src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Mon Oct 28 11:50:14 2013 +0200 +++ b/src/doveadm/client-connection.c Mon Oct 28 11:50:40 2013 +0200 @@ -263,7 +263,7 @@ /* FIXME: some day we should probably let auth process do this and support all kinds of authentication */ if (strncmp(line, "PLAIN\t", 6) != 0) { - i_error("doveadm client attempted non-PLAIN authentication"); + i_error("doveadm client attempted non-PLAIN authentication: %s", line); return -1; } From dovecot at dovecot.org Tue Oct 29 17:32:48 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 17:32:48 +0200 Subject: dovecot-2.2: lib-http: Added http_url_escape_path() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/37ebef9f641f changeset: 16890:37ebef9f641f user: Timo Sirainen date: Tue Oct 29 17:32:42 2013 +0200 description: lib-http: Added http_url_escape_path() diffstat: src/lib-http/http-url.c | 5 +++++ src/lib-http/http-url.h | 1 + 2 files changed, 6 insertions(+), 0 deletions(-) diffs (26 lines): diff -r 2599056381cc -r 37ebef9f641f src/lib-http/http-url.c --- a/src/lib-http/http-url.c Mon Oct 28 11:50:40 2013 +0200 +++ b/src/lib-http/http-url.c Tue Oct 29 17:32:42 2013 +0200 @@ -552,6 +552,11 @@ return str_c(urlstr); } +void http_url_escape_path(string_t *out, const char *data) +{ + uri_append_query_data(out, "&;?=+", data); +} + void http_url_escape_param(string_t *out, const char *data) { uri_append_query_data(out, "&;/?=+", data); diff -r 2599056381cc -r 37ebef9f641f src/lib-http/http-url.h --- a/src/lib-http/http-url.h Mon Oct 28 11:50:40 2013 +0200 +++ b/src/lib-http/http-url.h Tue Oct 29 17:32:42 2013 +0200 @@ -71,6 +71,7 @@ const char *http_url_create_authority(const struct http_url *url); const char *http_url_create_target(const struct http_url *url); +void http_url_escape_path(string_t *out, const char *data); void http_url_escape_param(string_t *out, const char *data); #endif From dovecot at dovecot.org Tue Oct 29 18:59:35 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 18:59:35 +0200 Subject: dovecot-2.2: virtual: If virtual mailbox has no config file, ass... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c120623cb109 changeset: 16891:c120623cb109 user: Timo Sirainen date: Tue Oct 29 18:59:20 2013 +0200 description: virtual: If virtual mailbox has no config file, assume it's just nonexistent. Most importantly if the virtual namespace root doesn't have the config, it should fail with MAIL_ERROR_NOTFOUND rather than MAIL_ERROR_NOTPOSSIBLE. diffstat: src/plugins/virtual/virtual-config.c | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diffs (21 lines): diff -r 37ebef9f641f -r c120623cb109 src/plugins/virtual/virtual-config.c --- a/src/plugins/virtual/virtual-config.c Tue Oct 29 17:32:42 2013 +0200 +++ b/src/plugins/virtual/virtual-config.c Tue Oct 29 18:59:20 2013 +0200 @@ -356,7 +356,6 @@ { struct mail_storage *storage = mbox->box.storage; struct virtual_parse_context ctx; - struct stat st; const char *box_path, *path, *line, *error; unsigned int linenum = 0; int fd, ret = 0; @@ -374,9 +373,6 @@ } else if (errno != ENOENT) { mail_storage_set_critical(storage, "open(%s) failed: %m", path); - } else if (stat(box_path, &st) == 0) { - mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE, - "Virtual mailbox missing configuration file"); } else if (errno == ENOENT) { mail_storage_set_error(storage, MAIL_ERROR_NOTFOUND, T_MAIL_ERR_MAILBOX_NOT_FOUND(mbox->box.vname)); From dovecot at dovecot.org Tue Oct 29 19:42:19 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 19:42:19 +0200 Subject: dovecot-2.2: imapc: Added a Courier-workaround for not returning... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c69d919edf63 changeset: 16892:c69d919edf63 user: Timo Sirainen date: Tue Oct 29 19:42:14 2013 +0200 description: imapc: Added a Courier-workaround for not returning UIDNEXT on SELECT. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 53 ++++++++++++++++++---------- 1 files changed, 33 insertions(+), 20 deletions(-) diffs (77 lines): diff -r c120623cb109 -r c69d919edf63 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Tue Oct 29 18:59:20 2013 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Tue Oct 29 19:42:14 2013 +0200 @@ -669,7 +669,7 @@ return index_storage_mailbox_delete(box); } -static int imapc_mailbox_get_status(struct mailbox *box, +static int imapc_mailbox_run_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status_r) { @@ -678,25 +678,6 @@ struct imapc_simple_context sctx; string_t *str; - if (mbox->guid_fetch_field_name != NULL || - IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED)) - status_r->have_guids = TRUE; - - if (box->opened) { - imapc_mailbox_get_selected_status(mbox, items, status_r); - return 0; - } - - /* mailbox isn't opened yet */ - if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS | - STATUS_PERMANENT_FLAGS)) != 0) { - /* getting these requires opening the mailbox */ - if (mailbox_open(box) < 0) - return -1; - imapc_mailbox_get_selected_status(mbox, items, status_r); - return 0; - } - str = t_str_new(256); if ((items & STATUS_MESSAGES) != 0) str_append(str, " MESSAGES"); @@ -726,6 +707,38 @@ return sctx.ret; } +static int imapc_mailbox_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r) +{ + struct imapc_mailbox *mbox = (struct imapc_mailbox *)box; + + if (mbox->guid_fetch_field_name != NULL || + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED)) + status_r->have_guids = TRUE; + + if (box->opened) { + imapc_mailbox_get_selected_status(mbox, items, status_r); + } else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS | + STATUS_PERMANENT_FLAGS)) != 0) { + /* getting these requires opening the mailbox */ + if (mailbox_open(box) < 0) + return -1; + imapc_mailbox_get_selected_status(mbox, items, status_r); + } else { + if (imapc_mailbox_run_status(box, items, status_r) < 0) + return -1; + } + + if (box->opened && (items & STATUS_UIDNEXT) != 0 && + mbox->sync_uid_next == 0) { + /* Courier-workaround, it doesn't send UIDNEXT on SELECT */ + if (imapc_mailbox_run_status(box, STATUS_UIDNEXT, status_r) < 0) + return -1; + } + return 0; +} + static int imapc_mailbox_get_namespaces(struct imapc_storage *storage) { enum imapc_capability capa; From dovecot at dovecot.org Tue Oct 29 20:26:43 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 20:26:43 +0200 Subject: dovecot-2.2: mbox: Added extra assert Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/41c844719166 changeset: 16893:41c844719166 user: Timo Sirainen date: Tue Oct 29 20:26:33 2013 +0200 description: mbox: Added extra assert diffstat: src/lib-storage/index/mbox/mbox-sync-rewrite.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c69d919edf63 -r 41c844719166 src/lib-storage/index/mbox/mbox-sync-rewrite.c --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Tue Oct 29 19:42:14 2013 +0200 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Tue Oct 29 20:26:33 2013 +0200 @@ -19,6 +19,7 @@ struct ostream *output; off_t ret; + i_assert(source > 0 || (dest != 1 && dest != 2)); i_assert(size < OFF_T_MAX); if (size == 0 || source == dest) From dovecot at dovecot.org Tue Oct 29 21:08:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 21:08:20 +0200 Subject: dovecot-2.2: lib-index: Don't mix index struct caching for in-me... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2b3948fa4a09 changeset: 16894:2b3948fa4a09 user: Timo Sirainen date: Tue Oct 29 21:07:28 2013 +0200 description: lib-index: Don't mix index struct caching for in-memory vs. disk indexes. diffstat: src/lib-index/mail-index-alloc-cache.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 41c844719166 -r 2b3948fa4a09 src/lib-index/mail-index-alloc-cache.c --- a/src/lib-index/mail-index-alloc-cache.c Tue Oct 29 20:26:33 2013 +0200 +++ b/src/lib-index/mail-index-alloc-cache.c Tue Oct 29 21:07:28 2013 +0200 @@ -97,7 +97,8 @@ else match = rec; } - } else if (mailbox_path != NULL && rec->mailbox_path != NULL) { + } else if (mailbox_path != NULL && rec->mailbox_path != NULL && + index_dir == NULL && rec->index_dir_ino == 0) { if (strcmp(mailbox_path, rec->mailbox_path) == 0) match = rec; } From dovecot at dovecot.org Tue Oct 29 21:08:20 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 21:08:20 +0200 Subject: dovecot-2.2: mdbox: Added "mdbox_deleted" storage, which can be ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c238c55479c8 changeset: 16895:c238c55479c8 user: Timo Sirainen date: Tue Oct 29 21:08:07 2013 +0200 description: mdbox: Added "mdbox_deleted" storage, which can be used to access messages with refcount=0 For example: doveadm import mdbox_deleted:~/mdbox "" mailbox inbox subject oops diffstat: src/lib-storage/index/dbox-multi/Makefile.am | 1 + src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c | 320 +++++++++++++++ src/lib-storage/index/dbox-multi/mdbox-map.c | 27 +- src/lib-storage/index/dbox-multi/mdbox-map.h | 8 + src/lib-storage/index/dbox-multi/mdbox-storage.c | 14 +- src/lib-storage/index/dbox-multi/mdbox-storage.h | 10 + 6 files changed, 369 insertions(+), 11 deletions(-) diffs (truncated from 499 to 300 lines): diff -r 2b3948fa4a09 -r c238c55479c8 src/lib-storage/index/dbox-multi/Makefile.am --- a/src/lib-storage/index/dbox-multi/Makefile.am Tue Oct 29 21:07:28 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/Makefile.am Tue Oct 29 21:08:07 2013 +0200 @@ -12,6 +12,7 @@ -I$(top_srcdir)/src/lib-storage/index/dbox-common libstorage_dbox_multi_la_SOURCES = \ + mdbox-deleted-storage.c \ mdbox-file.c \ mdbox-mail.c \ mdbox-map.c \ diff -r 2b3948fa4a09 -r c238c55479c8 src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-multi/mdbox-deleted-storage.c Tue Oct 29 21:08:07 2013 +0200 @@ -0,0 +1,320 @@ +/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "mkdir-parents.h" +#include "master-service.h" +#include "mail-index-modseq.h" +#include "mail-index-alloc-cache.h" +#include "mailbox-log.h" +#include "mailbox-list-private.h" +#include "mail-copy.h" +#include "dbox-mail.h" +#include "dbox-save.h" +#include "mdbox-map.h" +#include "mdbox-file.h" +#include "mdbox-sync.h" +#include "mdbox-storage-rebuild.h" +#include "mdbox-storage.h" + +extern struct mail_storage mdbox_deleted_storage; +extern struct mailbox mdbox_deleted_mailbox; +extern struct dbox_storage_vfuncs mdbox_deleted_dbox_storage_vfuncs; + +static struct mail_storage *mdbox_deleted_storage_alloc(void) +{ + struct mdbox_storage *storage; + pool_t pool; + + pool = pool_alloconly_create("mdbox deleted storage", 2048); + storage = p_new(pool, struct mdbox_storage, 1); + storage->storage.v = mdbox_dbox_storage_vfuncs; + storage->storage.storage = mdbox_deleted_storage; + storage->storage.storage.pool = pool; + return &storage->storage.storage; +} + +static struct mailbox * +mdbox_deleted_mailbox_alloc(struct mail_storage *storage, + struct mailbox_list *list, + const char *vname, enum mailbox_flags flags) +{ + struct mdbox_mailbox *mbox; + pool_t pool; + + flags |= MAILBOX_FLAG_READONLY | MAILBOX_FLAG_NO_INDEX_FILES; + + pool = pool_alloconly_create("mdbox deleted mailbox", 1024*3); + mbox = p_new(pool, struct mdbox_mailbox, 1); + mbox->box = mdbox_deleted_mailbox; + mbox->box.pool = pool; + mbox->box.storage = storage; + mbox->box.list = list; + mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; + + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); + + mbox->storage = (struct mdbox_storage *)storage; + return &mbox->box; +} + +static int +mdbox_deleted_mailbox_create_indexes(struct mailbox *box, + const struct mailbox_update *update, + struct mail_index_transaction *trans) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + struct mail_index_transaction *new_trans = NULL; + uint32_t uid_validity = ioloop_time; + uint32_t uid_next = 1; + + if (update != NULL && update->uid_validity != 0) + uid_validity = update->uid_validity; + + if (trans == NULL) { + new_trans = mail_index_transaction_begin(box->view, 0); + trans = new_trans; + } + + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &uid_next, sizeof(uid_next), TRUE); + mbox->creating = TRUE; + mdbox_update_header(mbox, trans, update); + mbox->creating = FALSE; + + if (new_trans != NULL) { + if (mail_index_transaction_commit(&new_trans) < 0) { + mailbox_set_index_error(box); + return -1; + } + } + return 0; +} + +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + +static int +mdbox_deleted_mailbox_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + if (index_mailbox_get_metadata(box, items, metadata_r) < 0) + return -1; + + if ((items & MAILBOX_METADATA_GUID) != 0) + guid_128_generate(metadata_r->guid); + return 0; +} + +static struct mail_save_context * +mdbox_deleted_save_alloc(struct mailbox_transaction_context *t) +{ + struct mail_save_context *ctx; + + ctx = i_new(struct mail_save_context, 1); + ctx->transaction = t; + return ctx; +} + +static int +mdbox_deleted_save_begin(struct mail_save_context *ctx, + struct istream *input ATTR_UNUSED) +{ + mail_storage_set_error(ctx->transaction->box->storage, + MAIL_ERROR_NOTPOSSIBLE, "mdbox_deleted doesn't support saving mails"); + return -1; +} + +static int +mdbox_deleted_save_continue(struct mail_save_context *ctx ATTR_UNUSED) +{ + return -1; +} + +static int mdbox_deleted_save_finish(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); + return -1; +} + +static void +mdbox_deleted_save_cancel(struct mail_save_context *ctx) +{ + index_save_context_free(ctx); +} + +static int mdbox_deleted_sync(struct mdbox_mailbox *mbox, + enum mdbox_sync_flags flags) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mdbox_mail_index_record rec; + struct mdbox_map_mail_index_record map_rec; + uint16_t refcount; + uint32_t map_seq, map_count, seq, uid = 0; + int ret = 0; + + if (mbox->mdbox_deleted_synced) { + /* don't bother supporting incremental syncs */ + return 0; + } + if (!mbox->box.inbox_user && mbox->box.name[0] != '\0') { + /* since mailbox list currently shows all the existing + mailboxes, we don't want all of them to list the deleted + messages. only show messages in user's INBOX or the + namespace prefix. */ + return 0; + } + + if (mdbox_map_open(mbox->storage->map) < 0) + return -1; + + if (mdbox_deleted_mailbox_create_indexes(&mbox->box, NULL, NULL) < 0) + return -1; + + memset(&rec, 0, sizeof(rec)); + rec.save_date = ioloop_time; + + if (mail_index_sync_begin(mbox->box.index, &index_sync_ctx, + &sync_view, &trans, flags) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + map_count = mdbox_map_get_messages_count(mbox->storage->map); + for (map_seq = 1; map_seq <= map_count; map_seq++) { + if (mdbox_map_lookup_seq_full(mbox->storage->map, map_seq, + &map_rec, &refcount) < 0) { + ret = -1; + break; + } + if (refcount == 0) { + rec.map_uid = mdbox_map_lookup_uid(mbox->storage->map, + map_seq); + mail_index_append(trans, ++uid, &seq); + mail_index_update_ext(trans, seq, + mbox->ext_id, &rec, NULL); + } + } + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else { + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } else { + mbox->mdbox_deleted_synced = TRUE; + } + } + return ret; +} + +static struct mailbox_sync_context * +mdbox_deleted_storage_sync_init(struct mailbox *box, + enum mailbox_sync_flags flags) +{ + struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; + enum mdbox_sync_flags mdbox_sync_flags = 0; + int ret = 0; + + if (!box->opened) { + if (mailbox_open(box) < 0) + ret = -1; + } + + if (ret == 0 && (index_mailbox_want_full_sync(&mbox->box, flags) || + mbox->storage->corrupted)) + ret = mdbox_deleted_sync(mbox, mdbox_sync_flags); + + return index_mailbox_sync_init(box, flags, ret < 0); +} + +struct mail_storage mdbox_deleted_storage = { + .name = MDBOX_DELETED_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_UNIQUE_ROOT | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUIDS | + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_SAVE_GUIDS | + MAIL_STORAGE_CLASS_FLAG_BINARY_DATA, + + .v = { + mdbox_get_setting_parser_info, + mdbox_deleted_storage_alloc, + mdbox_storage_create, + mdbox_storage_destroy, + NULL, + dbox_storage_get_list_settings, + NULL, + mdbox_deleted_mailbox_alloc, + NULL + } +}; + +struct mailbox mdbox_deleted_mailbox = { + .v = { + index_storage_is_readonly, + index_storage_mailbox_enable, + index_storage_mailbox_exists, + mdbox_mailbox_open, + index_storage_mailbox_close, + index_storage_mailbox_free, + dbox_mailbox_create, + index_storage_mailbox_update, + index_storage_mailbox_delete, + index_storage_mailbox_rename, + index_storage_get_status, + mdbox_deleted_mailbox_get_metadata, + index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, + index_storage_list_index_has_changed, + index_storage_list_index_update_sync, + mdbox_deleted_storage_sync_init, From dovecot at dovecot.org Tue Oct 29 21:11:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 21:11:16 +0200 Subject: dovecot-2.2: configure: Added mdbox_deleted storage. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e00ad71ee548 changeset: 16896:e00ad71ee548 user: Timo Sirainen date: Tue Oct 29 21:10:33 2013 +0200 description: configure: Added mdbox_deleted storage. diffstat: configure.ac | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (29 lines): diff -r c238c55479c8 -r e00ad71ee548 configure.ac --- a/configure.ac Tue Oct 29 21:08:07 2013 +0200 +++ b/configure.ac Tue Oct 29 21:10:33 2013 +0200 @@ -2523,11 +2523,15 @@ mailbox_list_drivers="maildir imapdir fs index none shared" have_sdbox=no +have_mdbox=no for storage in $mail_storages; do LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS `eval echo \\$${storage}_libs`" if test $storage = sdbox; then have_sdbox=yes fi + if test $storage = mdbox; then + have_mdbox=yes + fi if test $storage = sdbox || test $storage = mdbox; then LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS $dbox_common_libs" dbox_common_libs="" @@ -2546,6 +2550,9 @@ # create alias for sdbox mail_storages="$mail_storages dbox" fi +if test $have_mdbox = yes; then + mail_storages="$mail_storages mdbox_deleted" +fi dnl ** dnl ** Shared libraries usage From dovecot at dovecot.org Tue Oct 29 21:11:16 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 21:11:16 +0200 Subject: dovecot-2.2: lib-storage: Allow storage name to contain '_' Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff4941acbfe2 changeset: 16897:ff4941acbfe2 user: Timo Sirainen date: Tue Oct 29 21:11:10 2013 +0200 description: lib-storage: Allow storage name to contain '_' diffstat: src/lib-storage/mail-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e00ad71ee548 -r ff4941acbfe2 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Tue Oct 29 21:10:33 2013 +0200 +++ b/src/lib-storage/mail-storage.c Tue Oct 29 21:11:10 2013 +0200 @@ -118,7 +118,7 @@ /* check if data is in driver:data format (eg. mbox:~/mail) */ p = *data; - while (i_isalnum(*p)) p++; + while (i_isalnum(*p) || *p == '_') p++; if (*p == ':' && p != *data) { /* no autodetection if the storage driver is given. */ From dovecot at dovecot.org Tue Oct 29 21:16:12 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 29 Oct 2013 21:16:12 +0200 Subject: dovecot-2.2: Compiling fix for old GCC Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2c09cd664a95 changeset: 16898:2c09cd664a95 user: Timo Sirainen date: Tue Oct 29 21:16:05 2013 +0200 description: Compiling fix for old GCC diffstat: src/lib-http/test-http-server.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r ff4941acbfe2 -r 2c09cd664a95 src/lib-http/test-http-server.c --- a/src/lib-http/test-http-server.c Tue Oct 29 21:11:10 2013 +0200 +++ b/src/lib-http/test-http-server.c Tue Oct 29 21:16:05 2013 +0200 @@ -125,7 +125,7 @@ fd_listen = net_listen(&my_ip, &port, 128); if (fd_listen == -1) i_fatal("listen(port=%u) failed: %m", port); - io_listen = io_add(fd_listen, IO_READ, client_accept, NULL); + io_listen = io_add(fd_listen, IO_READ, client_accept, (void *)NULL); io_loop_run(ioloop); From dovecot at dovecot.org Wed Oct 30 12:12:06 2013 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 30 Oct 2013 12:12:06 +0200 Subject: dovecot-2.2: module-dir: Allow plugins to have deinit() function... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/70f1d2bb0bd9 changeset: 16899:70f1d2bb0bd9 user: Timo Sirainen date: Wed Oct 30 12:11:54 2013 +0200 description: module-dir: Allow plugins to have deinit() function without init() function diffstat: src/lib/module-dir.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (28 lines): diff -r 2c09cd664a95 -r 70f1d2bb0bd9 src/lib/module-dir.c --- a/src/lib/module-dir.c Tue Oct 29 21:16:05 2013 +0200 +++ b/src/lib/module-dir.c Wed Oct 30 12:11:54 2013 +0200 @@ -242,7 +242,7 @@ module->init = (void (*)(struct module *)) get_symbol(module, t_strconcat(name, "_init", NULL), !set->require_init_funcs); - module->deinit = module->init == NULL ? NULL : (void (*)(void)) + module->deinit = (void (*)(void)) get_symbol(module, t_strconcat(name, "_deinit", NULL), !set->require_init_funcs); @@ -492,10 +492,12 @@ struct module *module; for (module = modules; module != NULL; module = module->next) { - if (module->init != NULL && !module->initialized) T_BEGIN { + if (!module->initialized) { module->initialized = TRUE; - module->init(module); - } T_END; + if (module->init != NULL) T_BEGIN { + module->init(module); + } T_END; + } } }