From dovecot at dovecot.org Tue Oct 2 21:36:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 21:36:49 +0300 Subject: dovecot-2.1: lmtp: Fixed hanging on proxying if remote server wa... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/38727d3e90ec changeset: 14741:38727d3e90ec user: Timo Sirainen date: Tue Oct 02 21:36:43 2012 +0300 description: lmtp: Fixed hanging on proxying if remote server was down. Patch by Jack Bates. diffstat: src/lmtp/lmtp-proxy.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (10 lines): diff -r 6cac808c4bd8 -r 38727d3e90ec src/lmtp/lmtp-proxy.c --- a/src/lmtp/lmtp-proxy.c Fri Sep 28 15:12:28 2012 +0300 +++ b/src/lmtp/lmtp-proxy.c Tue Oct 02 21:36:43 2012 +0300 @@ -300,4 +300,6 @@ lmtp_client_send(conn->client, conn->data_input); lmtp_client_send_more(conn->client); } + /* finish if all of the connections have already failed */ + lmtp_proxy_try_finish(proxy); } From dovecot at dovecot.org Tue Oct 2 21:56:19 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 21:56:19 +0300 Subject: dovecot-2.1: mbox: Fixed getting filesystem permissions when par... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/83695d6d41aa changeset: 14742:83695d6d41aa user: Timo Sirainen date: Tue Oct 02 21:56:09 2012 +0300 description: mbox: Fixed getting filesystem permissions when parent dir has setgid-bit enabled. diffstat: src/lib-storage/mailbox-list.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (34 lines): diff -r 38727d3e90ec -r 83695d6d41aa src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue Oct 02 21:36:43 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Tue Oct 02 21:56:09 2012 +0300 @@ -638,7 +638,7 @@ void mailbox_list_get_permissions(struct mailbox_list *list, const char *name, struct mailbox_permissions *permissions_r) { - const char *path, *parent_name, *p; + const char *path, *parent_name, *parent_path, *p; struct stat st; memset(permissions_r, 0, sizeof(*permissions_r)); @@ -708,6 +708,21 @@ } else { permissions_r->file_create_gid = st.st_gid; } + if (!S_ISDIR(st.st_mode) && + permissions_r->file_create_gid != (gid_t)-1) { + /* we need to stat() the parent directory to see if + it has setgid-bit set */ + p = strrchr(path, '/'); + parent_path = p == NULL ? NULL : + t_strdup_until(path, p); + if (parent_path != NULL && + stat(parent_path, &st) == 0 && + (st.st_mode & S_ISGID) != 0) { + /* directory's GID is used automatically for + new files */ + permissions_r->file_create_gid = (gid_t)-1; + } + } } if (name == NULL) { From dovecot at dovecot.org Tue Oct 2 22:38:00 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 22:38:00 +0300 Subject: dovecot-2.1: doveadm: Fixed printing large input from doveadm-se... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/94c7e875f9b9 changeset: 14743:94c7e875f9b9 user: Timo Sirainen date: Tue Oct 02 22:37:49 2012 +0300 description: doveadm: Fixed printing large input from doveadm-server. diffstat: src/doveadm/server-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 83695d6d41aa -r 94c7e875f9b9 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Tue Oct 02 21:56:09 2012 +0300 +++ b/src/doveadm/server-connection.c Tue Oct 02 22:37:49 2012 +0300 @@ -100,7 +100,8 @@ { if (conn->streaming) { conn->streaming = FALSE; - stream_data(str, data, size); + if (size > 0) + stream_data(str, data, size); doveadm_print_stream("", 0); } else { const char *text; From dovecot at dovecot.org Tue Oct 2 22:45:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 22:45:44 +0300 Subject: dovecot-2.2: lib-imap: Minor fix to imap_url_parse() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc512eba5207 changeset: 15171:fc512eba5207 user: Timo Sirainen date: Tue Oct 02 22:45:34 2012 +0300 description: lib-imap: Minor fix to imap_url_parse() diffstat: src/lib-imap/imap-url.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b6663a4f60fe -r fc512eba5207 src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Fri Sep 28 15:11:54 2012 +0300 +++ b/src/lib-imap/imap-url.c Tue Oct 02 22:45:34 2012 +0300 @@ -864,7 +864,7 @@ /* [ "?" enc-search ] */ if ((ret = uri_parse_query(parser, &query)) != 0) { if (ret < 0) - return ret; + return FALSE; if (!is_messagelist) { parser->error = From dovecot at dovecot.org Tue Oct 2 22:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 22:55:54 +0300 Subject: dovecot-2.2: lib: Generalize hmac to be hash independent Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8802322d7257 changeset: 15172:8802322d7257 user: Florian Zeitz date: Thu Aug 30 00:43:56 2012 +0200 description: lib: Generalize hmac to be hash independent diffstat: src/auth/auth-token.c | 18 +++-- src/auth/mech-cram-md5.c | 11 ++- src/auth/mech-scram-sha1.c | 51 +++++++++-------- src/auth/password-scheme.c | 9 +- src/lib-imap-urlauth/imap-urlauth.c | 11 ++- src/lib-ntlm/ntlm-encrypt.c | 23 ++++--- src/lib/Makefile.am | 8 +- src/lib/hmac-cram-md5.c | 63 ++++++++++++++++++++++ src/lib/hmac-cram-md5.h | 14 ++++ src/lib/hmac-md5.c | 101 ------------------------------------ src/lib/hmac-md5.h | 29 ---------- src/lib/hmac-sha1.c | 52 ------------------ src/lib/hmac-sha1.h | 22 ------- src/lib/hmac.c | 58 ++++++++++++++++++++ src/lib/hmac.h | 26 +++++++++ 15 files changed, 231 insertions(+), 265 deletions(-) diffs (truncated from 740 to 300 lines): diff -r fc512eba5207 -r 8802322d7257 src/auth/auth-token.c --- a/src/auth/auth-token.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/auth/auth-token.c Thu Aug 30 00:43:56 2012 +0200 @@ -11,7 +11,8 @@ #include "auth-common.h" #include "hex-binary.h" -#include "hmac-sha1.h" +#include "hmac.h" +#include "sha1.h" #include "randgen.h" #include "read-full.h" #include "write-full.h" @@ -168,16 +169,17 @@ const char *auth_token_get(const char *service, const char *session_pid, const char *username, const char *session_id) { - struct hmac_sha1_context ctx; + struct hmac_context ctx; unsigned char result[SHA1_RESULTLEN]; - hmac_sha1_init(&ctx, username, strlen(username)); - hmac_sha1_update(&ctx, session_pid, strlen(session_pid)); + hmac_init(&ctx, (const unsigned char*)username, strlen(username), + &hash_method_sha1); + hmac_update(&ctx, session_pid, strlen(session_pid)); if (session_id != NULL && *session_id != '\0') - hmac_sha1_update(&ctx, session_id, strlen(session_id)); - hmac_sha1_update(&ctx, service, strlen(service)); - hmac_sha1_update(&ctx, auth_token_secret, sizeof(auth_token_secret)); - hmac_sha1_final(&ctx, result); + hmac_update(&ctx, session_id, strlen(session_id)); + hmac_update(&ctx, service, strlen(service)); + hmac_update(&ctx, auth_token_secret, sizeof(auth_token_secret)); + hmac_final(&ctx, result); return binary_to_hex(result, sizeof(result)); } diff -r fc512eba5207 -r 8802322d7257 src/auth/mech-cram-md5.c --- a/src/auth/mech-cram-md5.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/auth/mech-cram-md5.c Thu Aug 30 00:43:56 2012 +0200 @@ -7,7 +7,9 @@ #include "ioloop.h" #include "buffer.h" #include "hex-binary.h" -#include "hmac-md5.h" +#include "hmac-cram-md5.h" +#include "hmac.h" +#include "md5.h" #include "randgen.h" #include "mech.h" #include "passdb.h" @@ -50,7 +52,7 @@ { unsigned char digest[MD5_RESULTLEN]; - struct hmac_md5_context ctx; + struct hmac_context ctx; const char *response_hex; if (size != CRAM_MD5_CONTEXTLEN) { @@ -59,9 +61,10 @@ return FALSE; } + hmac_init(&ctx, NULL, 0, &hash_method_md5); hmac_md5_set_cram_context(&ctx, credentials); - hmac_md5_update(&ctx, request->challenge, strlen(request->challenge)); - hmac_md5_final(&ctx, digest); + hmac_update(&ctx, request->challenge, strlen(request->challenge)); + hmac_final(&ctx, digest); response_hex = binary_to_hex(digest, sizeof(digest)); diff -r fc512eba5207 -r 8802322d7257 src/auth/mech-scram-sha1.c --- a/src/auth/mech-scram-sha1.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/auth/mech-scram-sha1.c Thu Aug 30 00:43:56 2012 +0200 @@ -9,7 +9,8 @@ #include "auth-common.h" #include "base64.h" #include "buffer.h" -#include "hmac-sha1.h" +#include "hmac.h" +#include "sha1.h" #include "randgen.h" #include "safe-memset.h" #include "str.h" @@ -44,23 +45,23 @@ const unsigned char *salt, size_t salt_size, unsigned int i, unsigned char result[SHA1_RESULTLEN]) { - struct hmac_sha1_context ctx; + struct hmac_context ctx; unsigned char U[SHA1_RESULTLEN]; unsigned int j, k; /* Calculate U1 */ - hmac_sha1_init(&ctx, str, str_size); - hmac_sha1_update(&ctx, salt, salt_size); - hmac_sha1_update(&ctx, "\0\0\0\1", 4); - hmac_sha1_final(&ctx, U); + hmac_init(&ctx, str, str_size, &hash_method_sha1); + hmac_update(&ctx, salt, salt_size); + hmac_update(&ctx, "\0\0\0\1", 4); + hmac_final(&ctx, U); memcpy(result, U, SHA1_RESULTLEN); /* Calculate U2 to Ui and Hi */ for (j = 2; j <= i; j++) { - hmac_sha1_init(&ctx, str, str_size); - hmac_sha1_update(&ctx, U, sizeof(U)); - hmac_sha1_final(&ctx, U); + hmac_init(&ctx, str, str_size, &hash_method_sha1); + hmac_update(&ctx, U, sizeof(U)); + hmac_final(&ctx, U); for (k = 0; k < SHA1_RESULTLEN; k++) result[k] ^= U[k]; } @@ -94,7 +95,7 @@ static const char *get_scram_server_final(struct scram_auth_request *request) { - struct hmac_sha1_context ctx; + struct hmac_context ctx; const char *auth_message; unsigned char server_key[SHA1_RESULTLEN]; unsigned char server_signature[SHA1_RESULTLEN]; @@ -104,17 +105,17 @@ request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_sha1_init(&ctx, request->salted_password, - sizeof(request->salted_password)); - hmac_sha1_update(&ctx, "Server Key", 10); - hmac_sha1_final(&ctx, server_key); + hmac_init(&ctx, request->salted_password, + sizeof(request->salted_password), &hash_method_sha1); + hmac_update(&ctx, "Server Key", 10); + hmac_final(&ctx, server_key); safe_memset(request->salted_password, 0, sizeof(request->salted_password)); - hmac_sha1_init(&ctx, server_key, sizeof(server_key)); - hmac_sha1_update(&ctx, auth_message, strlen(auth_message)); - hmac_sha1_final(&ctx, server_signature); + hmac_init(&ctx, server_key, sizeof(server_key), &hash_method_sha1); + hmac_update(&ctx, auth_message, strlen(auth_message)); + hmac_final(&ctx, server_signature); str = t_str_new(MAX_BASE64_ENCODED_SIZE(sizeof(server_signature))); str_append(str, "v="); @@ -213,7 +214,7 @@ static bool verify_credentials(struct scram_auth_request *request, const unsigned char *credentials, size_t size) { - struct hmac_sha1_context ctx; + struct hmac_context ctx; const char *auth_message; unsigned char client_key[SHA1_RESULTLEN]; unsigned char client_signature[SHA1_RESULTLEN]; @@ -224,10 +225,10 @@ Hi(credentials, size, request->salt, sizeof(request->salt), SCRAM_ITERATE_COUNT, request->salted_password); - hmac_sha1_init(&ctx, request->salted_password, - sizeof(request->salted_password)); - hmac_sha1_update(&ctx, "Client Key", 10); - hmac_sha1_final(&ctx, client_key); + hmac_init(&ctx, request->salted_password, + sizeof(request->salted_password), &hash_method_sha1); + hmac_update(&ctx, "Client Key", 10); + hmac_final(&ctx, client_key); sha1_get_digest(client_key, sizeof(client_key), stored_key); @@ -235,9 +236,9 @@ request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_sha1_init(&ctx, stored_key, sizeof(stored_key)); - hmac_sha1_update(&ctx, auth_message, strlen(auth_message)); - hmac_sha1_final(&ctx, client_signature); + hmac_init(&ctx, stored_key, sizeof(stored_key), &hash_method_sha1); + hmac_update(&ctx, auth_message, strlen(auth_message)); + hmac_final(&ctx, client_signature); for (i = 0; i < sizeof(client_signature); i++) client_signature[i] ^= client_key[i]; diff -r fc512eba5207 -r 8802322d7257 src/auth/password-scheme.c --- a/src/auth/password-scheme.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/auth/password-scheme.c Thu Aug 30 00:43:56 2012 +0200 @@ -6,7 +6,8 @@ #include "hex-binary.h" #include "md4.h" #include "md5.h" -#include "hmac-md5.h" +#include "hmac.h" +#include "hmac-cram-md5.h" #include "ntlm.h" #include "mycrypt.h" #include "randgen.h" @@ -655,12 +656,12 @@ cram_md5_generate(const char *plaintext, const char *user ATTR_UNUSED, const unsigned char **raw_password_r, size_t *size_r) { - struct hmac_md5_context ctx; + struct hmac_context ctx; unsigned char *context_digest; context_digest = t_malloc(CRAM_MD5_CONTEXTLEN); - hmac_md5_init(&ctx, (const unsigned char *)plaintext, - strlen(plaintext)); + hmac_init(&ctx, (const unsigned char *)plaintext, + strlen(plaintext), &hash_method_md5); hmac_md5_get_cram_context(&ctx, context_digest); *raw_password_r = context_digest; diff -r fc512eba5207 -r 8802322d7257 src/lib-imap-urlauth/imap-urlauth.c --- a/src/lib-imap-urlauth/imap-urlauth.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth.c Thu Aug 30 00:43:56 2012 +0200 @@ -3,7 +3,8 @@ #include "lib.h" #include "hostpid.h" #include "var-expand.h" -#include "hmac-sha1.h" +#include "hmac.h" +#include "sha1.h" #include "randgen.h" #include "safe-memset.h" #include "mail-storage.h" @@ -88,15 +89,15 @@ const unsigned char mailbox_key[IMAP_URLAUTH_KEY_LEN], size_t *token_len_r) { - struct hmac_sha1_context hmac; + struct hmac_context hmac; unsigned char *token; token = t_new(unsigned char, SHA1_RESULTLEN + 1); token[0] = IMAP_URLAUTH_MECH_INTERNAL_VERSION; - hmac_sha1_init(&hmac, mailbox_key, IMAP_URLAUTH_KEY_LEN); - hmac_sha1_update(&hmac, rumpurl, strlen(rumpurl)); - hmac_sha1_final(&hmac, token+1); + hmac_init(&hmac, mailbox_key, IMAP_URLAUTH_KEY_LEN, &hash_method_sha1); + hmac_update(&hmac, rumpurl, strlen(rumpurl)); + hmac_final(&hmac, token+1); *token_len_r = SHA1_RESULTLEN + 1; return token; diff -r fc512eba5207 -r 8802322d7257 src/lib-ntlm/ntlm-encrypt.c --- a/src/lib-ntlm/ntlm-encrypt.c Tue Oct 02 22:45:34 2012 +0300 +++ b/src/lib-ntlm/ntlm-encrypt.c Thu Aug 30 00:43:56 2012 +0200 @@ -11,7 +11,8 @@ #include "compat.h" #include "safe-memset.h" #include "md4.h" -#include "hmac-md5.h" +#include "md5.h" +#include "hmac.h" #include "ntlm.h" #include "ntlm-des.h" @@ -60,12 +61,12 @@ } static void -hmac_md5_ucs2le_string_ucase(struct hmac_md5_context *ctx, const char *str) +hmac_md5_ucs2le_string_ucase(struct hmac_context *ctx, const char *str) { size_t len; unsigned char *wstr = t_unicode_str(str, 1, &len); - hmac_md5_update(ctx, wstr, len); + hmac_update(ctx, wstr, len); } static void ATTR_NULL(2) @@ -73,13 +74,13 @@ const unsigned char *hash_v1, unsigned char hash[NTLMSSP_V2_HASH_SIZE]) { - struct hmac_md5_context ctx; + struct hmac_context ctx; - hmac_md5_init(&ctx, hash_v1, NTLMSSP_HASH_SIZE); + hmac_init(&ctx, hash_v1, NTLMSSP_HASH_SIZE, &hash_method_md5); hmac_md5_ucs2le_string_ucase(&ctx, user); if (target != NULL) hmac_md5_ucs2le_string_ucase(&ctx, target); - hmac_md5_final(&ctx, hash); + hmac_final(&ctx, hash); } void @@ -124,15 +125,15 @@ const unsigned char *blob, size_t blob_size, unsigned char response[NTLMSSP_V2_RESPONSE_SIZE]) From dovecot at dovecot.org Tue Oct 2 23:12:14 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 23:12:14 +0300 Subject: dovecot-2.1: lib-master: Fixed crashes with settings cache. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e29b627219b3 changeset: 14744:e29b627219b3 user: Timo Sirainen date: Tue Oct 02 23:12:07 2012 +0300 description: lib-master: Fixed crashes with settings cache. diffstat: src/lib-master/master-service-settings-cache.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 94c7e875f9b9 -r e29b627219b3 src/lib-master/master-service-settings-cache.c --- a/src/lib-master/master-service-settings-cache.c Tue Oct 02 22:37:49 2012 +0300 +++ b/src/lib-master/master-service-settings-cache.c Tue Oct 02 23:12:07 2012 +0300 @@ -83,6 +83,7 @@ } for (entry = cache->oldest; entry != NULL; entry = next) { next = entry->next; + i_assert(entry->parser != cache->global_parser); settings_parser_deinit(&entry->parser); pool_unref(&entry->pool); } @@ -146,8 +147,10 @@ } if (entry != NULL) { - DLLIST2_REMOVE(&cache->oldest, &cache->newest, entry); - DLLIST2_APPEND(&cache->oldest, &cache->newest, entry); + if (entry->parser != cache->global_parser) { + DLLIST2_REMOVE(&cache->oldest, &cache->newest, entry); + DLLIST2_APPEND(&cache->oldest, &cache->newest, entry); + } *parser_r = entry->parser; return TRUE; } From dovecot at dovecot.org Tue Oct 2 23:24:15 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 02 Oct 2012 23:24:15 +0300 Subject: dovecot-2.1: lib-storage: When index mkdir() fails with EPERM, c... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3ce50c0bb782 changeset: 14745:3ce50c0bb782 user: Timo Sirainen date: Tue Oct 02 23:24:10 2012 +0300 description: lib-storage: When index mkdir() fails with EPERM, create the dir anyway with 0700 mode. This avoids failing entirely when /var/mail/user has 0660 permissions and we don't have access to the group. The error message is still logged. diffstat: src/lib-storage/mailbox-list.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r e29b627219b3 -r 3ce50c0bb782 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue Oct 02 23:12:07 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Tue Oct 02 23:24:10 2012 +0300 @@ -1461,7 +1461,14 @@ if (errno != ENOENT || p == NULL || ++n == 2) { mailbox_list_set_critical(list, "mkdir(%s) failed: %m", index_dir); - return -1; + if (p == NULL || errno != EPERM || + perm.dir_create_mode == 0700) + return -1; + /* we can't use the GID. allow it anyway with more + restricted permissions. */ + perm.file_create_gid = (gid_t)-1; + perm.dir_create_mode = 0700; + continue; } /* create the parent directory first */ parent_dir = t_strdup_until(index_dir, p); From dovecot at dovecot.org Wed Oct 3 00:41:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 00:41:22 +0300 Subject: dovecot-2.2: auth: Don't add proxy/pass fields when we're only a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff66315076ce changeset: 15173:ff66315076ce user: Timo Sirainen date: Wed Oct 03 00:41:18 2012 +0300 description: auth: Don't add proxy/pass fields when we're only authenticating (not logging in). For example SMTP server doesn't need these fields when doing SMTP AUTH. diffstat: src/auth/auth-request-handler.c | 3 ++- src/auth/auth-request.c | 2 +- src/auth/auth-request.h | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diffs (43 lines): diff -r 8802322d7257 -r ff66315076ce src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Thu Aug 30 00:43:56 2012 +0200 +++ b/src/auth/auth-request-handler.c Wed Oct 03 00:41:18 2012 +0300 @@ -181,7 +181,7 @@ } } - if (request->proxy) { + if (request->proxy && !request->auth_only) { /* we're proxying */ if (!seen_pass && request->mech_password != NULL) { /* send back the password that was sent by user @@ -490,6 +490,7 @@ request->connect_uid = handler->connect_uid; request->client_pid = handler->client_pid; request->id = id; + request->auth_only = handler->master_callback == NULL; /* parse optional parameters */ initial_resp = NULL; diff -r 8802322d7257 -r ff66315076ce src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Aug 30 00:43:56 2012 +0200 +++ b/src/auth/auth-request.c Wed Oct 03 00:41:18 2012 +0300 @@ -1633,7 +1633,7 @@ { int ret; - if (!request->proxy) + if (!request->proxy || request->auth_only) return 1; if ((ret = auth_request_proxy_host_lookup(request, callback)) <= 0) diff -r 8802322d7257 -r ff66315076ce src/auth/auth-request.h --- a/src/auth/auth-request.h Thu Aug 30 00:43:56 2012 +0200 +++ b/src/auth/auth-request.h Wed Oct 03 00:41:18 2012 +0300 @@ -101,6 +101,7 @@ unsigned int passdb_internal_failure:1; unsigned int userdb_internal_failure:1; unsigned int delayed_failure:1; + unsigned int auth_only:1; unsigned int domain_is_realm:1; unsigned int accept_input:1; unsigned int no_failure_delay:1; From dovecot at dovecot.org Wed Oct 3 01:26:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 01:26:52 +0300 Subject: dovecot-2.1: lib-storage: struct mail.close() now clears all of ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/4c3dac1a94cf changeset: 14747:4c3dac1a94cf user: Timo Sirainen date: Wed Oct 03 01:19:19 2012 +0300 description: lib-storage: struct mail.close() now clears all of its data. diffstat: src/lib-storage/index/index-mail.c | 51 +++++++++++++++++++------------------ src/lib-storage/index/index-mail.h | 2 + 2 files changed, 28 insertions(+), 25 deletions(-) diffs (104 lines): diff -r 8c06fe0b280d -r 4c3dac1a94cf src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Oct 03 01:12:13 2012 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Oct 03 01:19:19 2012 +0300 @@ -1180,32 +1180,10 @@ index_mail_close_streams_full(mail, FALSE); } -void index_mail_close(struct mail *_mail) -{ - struct index_mail *mail = (struct index_mail *)_mail; - - /* If uid == 0 but seq != 0, we came here from saving a (non-mbox) - message. If that happens, don't bother checking if anything should - be cached since it was already checked. Also by now the transaction - may have already been rollbacked and seq point to a nonexistent - message. */ - if (mail->mail.mail.uid != 0) { - index_mail_cache_sizes(mail); - index_mail_cache_dates(mail); - } - - index_mail_close_streams_full(mail, TRUE); - - if (mail->data.wanted_headers != NULL) - mailbox_header_lookup_unref(&mail->data.wanted_headers); -} - -static void index_mail_reset(struct index_mail *mail) +static void index_mail_reset_data(struct index_mail *mail) { struct index_mail_data *data = &mail->data; - mail->mail.v.close(&mail->mail.mail); - memset(data, 0, sizeof(*data)); p_clear(mail->data_pool); @@ -1230,6 +1208,28 @@ mail->mail.mail.saving = FALSE; } +void index_mail_close(struct mail *_mail) +{ + struct index_mail *mail = (struct index_mail *)_mail; + + /* If uid == 0 but seq != 0, we came here from saving a (non-mbox) + message. If that happens, don't bother checking if anything should + be cached since it was already checked. Also by now the transaction + may have already been rollbacked and seq point to a nonexistent + message. */ + if (mail->mail.mail.uid != 0) { + index_mail_cache_sizes(mail); + index_mail_cache_dates(mail); + } + + index_mail_close_streams_full(mail, TRUE); + + if (mail->data.wanted_headers != NULL) + mailbox_header_lookup_unref(&mail->data.wanted_headers); + if (!mail->freeing) + index_mail_reset_data(mail); +} + static void check_envelope(struct index_mail *mail) { struct mail *_mail = &mail->mail.mail; @@ -1390,7 +1390,7 @@ if (mail->data.seq == seq) return; - index_mail_reset(mail); + mail->mail.v.close(&mail->mail.mail); mail->data.seq = seq; mail->mail.mail.seq = seq; @@ -1458,7 +1458,7 @@ index_mail_set_seq(_mail, seq, FALSE); return TRUE; } else { - index_mail_reset(mail); + mail->mail.v.close(&mail->mail.mail); mail->mail.mail.uid = uid; mail_set_expunged(&mail->mail.mail); return FALSE; @@ -1512,6 +1512,7 @@ directly */ i_assert(!mail->search_mail); + mail->freeing = TRUE; mail->mail.v.close(_mail); i_assert(_mail->transaction->mail_ref_count > 0); diff -r 8c06fe0b280d -r 4c3dac1a94cf src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Wed Oct 03 01:12:13 2012 +0300 +++ b/src/lib-storage/index/index-mail.h Wed Oct 03 01:19:19 2012 +0300 @@ -146,6 +146,8 @@ unsigned int pop3_state_set:1; /* mail created by mailbox_search_*() */ unsigned int search_mail:1; + /* close() is being called from mail_free() */ + unsigned int freeing:1; }; struct mail * From dovecot at dovecot.org Wed Oct 3 01:26:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 01:26:52 +0300 Subject: dovecot-2.1: dbox: Small code cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8c06fe0b280d changeset: 14746:8c06fe0b280d user: Timo Sirainen date: Wed Oct 03 01:12:13 2012 +0300 description: dbox: Small code cleanup. diffstat: src/lib-storage/index/dbox-common/dbox-save.c | 15 ++++++++------- src/lib-storage/index/dbox-multi/mdbox-save.c | 2 -- src/lib-storage/index/dbox-single/sdbox-save.c | 6 +----- 3 files changed, 9 insertions(+), 14 deletions(-) diffs (55 lines): diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-common/dbox-save.c --- a/src/lib-storage/index/dbox-common/dbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -105,13 +105,14 @@ if (index_attachment_save_finish(&ctx->ctx) < 0) ctx->failed = TRUE; } - if (ctx->ctx.output == dbox_output) - return; - - /* e.g. zlib plugin had changed this */ - o_stream_ref(dbox_output); - o_stream_destroy(&ctx->ctx.output); - ctx->ctx.output = dbox_output; + if (ctx->ctx.output != dbox_output) { + /* e.g. zlib plugin had changed this */ + o_stream_ref(dbox_output); + o_stream_destroy(&ctx->ctx.output); + ctx->ctx.output = dbox_output; + } + index_mail_cache_parse_deinit(ctx->ctx.dest_mail, + ctx->ctx.received_date, !ctx->failed); } void dbox_save_write_metadata(struct mail_save_context *_ctx, diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -199,8 +199,6 @@ return -1; dbox_save_end(&ctx->ctx); - index_mail_cache_parse_deinit(_ctx->dest_mail, - _ctx->received_date, !ctx->ctx.failed); mail = array_idx_modifiable(&ctx->mails, array_count(&ctx->mails) - 1); if (!ctx->ctx.failed) T_BEGIN { diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-single/sdbox-save.c --- a/src/lib-storage/index/dbox-single/sdbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -181,13 +181,9 @@ index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); } - - index_mail_cache_parse_deinit(_ctx->dest_mail, - _ctx->received_date, !ctx->ctx.failed); + dbox_save_end(&ctx->ctx); files = array_idx_modifiable(&ctx->files, array_count(&ctx->files) - 1); - - dbox_save_end(&ctx->ctx); if (!ctx->ctx.failed) T_BEGIN { if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0) ctx->ctx.failed = TRUE; From dovecot at dovecot.org Wed Oct 3 01:26:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 01:26:52 +0300 Subject: dovecot-2.1: lib-storage: mailbox_save_cancel() now makes sure t... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3de5a5b49580 changeset: 14748:3de5a5b49580 user: Timo Sirainen date: Wed Oct 03 01:20:22 2012 +0300 description: lib-storage: mailbox_save_cancel() now makes sure that dest_mail is reset. This fixes e.g. doveadm import, which continues import even though some messages couldn't be saved. diffstat: src/lib-storage/mail-storage.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (23 lines): diff -r 4c3dac1a94cf -r 3de5a5b49580 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Wed Oct 03 01:19:19 2012 +0300 +++ b/src/lib-storage/mail-storage.c Wed Oct 03 01:20:22 2012 +0300 @@ -1680,11 +1680,19 @@ { struct mail_save_context *ctx = *_ctx; struct mail_keywords *keywords = ctx->keywords; + struct mail_private *mail; *_ctx = NULL; ctx->transaction->box->v.save_cancel(ctx); if (keywords != NULL) mailbox_keywords_unref(&keywords); + if (ctx->dest_mail != NULL) { + /* the dest_mail is no longer valid. if we're still saving + more mails, the mail sequence may get reused. make sure + the mail gets reset in between */ + mail = (struct mail_private *)ctx->dest_mail; + mail->v.close(&mail->mail); + } } struct mailbox_transaction_context * From dovecot at dovecot.org Wed Oct 3 01:26:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 01:26:52 +0300 Subject: dovecot-2.1: mdbox: Fix to handling transactions with partially ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/19732dd98602 changeset: 14749:19732dd98602 user: Timo Sirainen date: Wed Oct 03 01:26:42 2012 +0300 description: mdbox: Fix to handling transactions with partially failed saves. diffstat: src/lib-storage/index/dbox-common/dbox-file.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 3de5a5b49580 -r 19732dd98602 src/lib-storage/index/dbox-common/dbox-file.c --- a/src/lib-storage/index/dbox-common/dbox-file.c Wed Oct 03 01:20:22 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Wed Oct 03 01:26:42 2012 +0300 @@ -532,7 +532,8 @@ { struct mail_storage *storage = &ctx->file->storage->storage; - if (ctx->last_flush_offset == ctx->output->offset) + if (ctx->last_flush_offset == ctx->output->offset && + ctx->last_checkpoint_offset == ctx->output->offset) return 0; if (o_stream_flush(ctx->output) < 0) { @@ -540,6 +541,14 @@ return -1; } + if (ctx->last_checkpoint_offset != ctx->output->offset) { + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + o_stream_seek(ctx->output, ctx->last_checkpoint_offset); + } + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { if (fdatasync(ctx->file->fd) < 0) { dbox_file_set_syscall_error(ctx->file, "fdatasync()"); From dovecot at dovecot.org Wed Oct 3 01:38:24 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 01:38:24 +0300 Subject: dovecot-2.1: sdbox: Fix to handling transactions with partially ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b99bead8b339 changeset: 14750:b99bead8b339 user: Timo Sirainen date: Wed Oct 03 01:38:20 2012 +0300 description: sdbox: Fix to handling transactions with partially failed saves. diffstat: src/lib-storage/index/dbox-single/sdbox-save.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (31 lines): diff -r 19732dd98602 -r b99bead8b339 src/lib-storage/index/dbox-single/sdbox-save.c --- a/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:26:42 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:38:20 2012 +0300 @@ -168,7 +168,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) { struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; - struct dbox_file *const *files; + struct dbox_file **files; ctx->ctx.finished = TRUE; if (ctx->ctx.dbox_output == NULL) @@ -192,14 +192,17 @@ if (ctx->ctx.failed) { mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq); dbox_file_append_rollback(&ctx->append_ctx); + dbox_file_unlink(*files); + dbox_file_unref(files); + array_delete(&ctx->files, array_count(&ctx->files) - 1, 1); } else { dbox_file_append_checkpoint(ctx->append_ctx); if (dbox_file_append_commit(&ctx->append_ctx) < 0) ctx->ctx.failed = TRUE; + dbox_file_close(*files); } i_stream_unref(&ctx->ctx.input); - dbox_file_close(*files); ctx->ctx.dbox_output = NULL; return ctx->ctx.failed ? -1 : 0; From dovecot at dovecot.org Wed Oct 3 02:29:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 02:29:27 +0300 Subject: dovecot-2.2: Added [io]_stream_create_error() for creating strea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f860cdf156cf changeset: 15174:f860cdf156cf user: Timo Sirainen date: Sun Sep 30 19:11:55 2012 +0300 description: Added [io]_stream_create_error() for creating streams that always fail reads/writes. diffstat: src/lib/istream.c | 15 +++++++++++++++ src/lib/istream.h | 1 + src/lib/ostream.c | 14 ++++++++++++++ src/lib/ostream.h | 2 ++ 4 files changed, 32 insertions(+), 0 deletions(-) diffs (66 lines): diff -r ff66315076ce -r f860cdf156cf src/lib/istream.c --- a/src/lib/istream.c Wed Oct 03 00:41:18 2012 +0300 +++ b/src/lib/istream.c Sun Sep 30 19:11:55 2012 +0300 @@ -685,3 +685,18 @@ io_stream_init(&_stream->iostream); return &_stream->istream; } + +struct istream *i_stream_create_error(int stream_errno) +{ + struct istream_private *stream; + + stream = i_new(struct istream_private, 1); + stream->istream.closed = TRUE; + stream->istream.readable_fd = FALSE; + stream->istream.blocking = TRUE; + stream->istream.seekable = TRUE; + stream->istream.stream_errno = stream_errno; + i_stream_create(stream, NULL, -1); + i_stream_set_name(&stream->istream, "(error)"); + return &stream->istream; +} diff -r ff66315076ce -r f860cdf156cf src/lib/istream.h --- a/src/lib/istream.h Wed Oct 03 00:41:18 2012 +0300 +++ b/src/lib/istream.h Sun Sep 30 19:11:55 2012 +0300 @@ -34,6 +34,7 @@ struct istream *i_stream_create_limit(struct istream *input, uoff_t v_size); struct istream *i_stream_create_range(struct istream *input, uoff_t v_offset, uoff_t v_size); +struct istream *i_stream_create_error(int stream_errno); /* Set name (e.g. path) for input stream. */ void i_stream_set_name(struct istream *stream, const char *name); diff -r ff66315076ce -r f860cdf156cf src/lib/ostream.c --- a/src/lib/ostream.c Wed Oct 03 00:41:18 2012 +0300 +++ b/src/lib/ostream.c Sun Sep 30 19:11:55 2012 +0300 @@ -512,3 +512,17 @@ io_stream_init(&_stream->iostream); return &_stream->ostream; } + +struct ostream *o_stream_create_error(int stream_errno) +{ + struct ostream_private *stream; + struct ostream *output; + + stream = i_new(struct ostream_private, 1); + stream->ostream.closed = TRUE; + stream->ostream.stream_errno = stream_errno; + + output = o_stream_create(stream, NULL, -1); + o_stream_set_name(output, "(error)"); + return output; +} diff -r ff66315076ce -r f860cdf156cf src/lib/ostream.h --- a/src/lib/ostream.h Wed Oct 03 00:41:18 2012 +0300 +++ b/src/lib/ostream.h Sun Sep 30 19:11:55 2012 +0300 @@ -36,6 +36,8 @@ o_stream_create_fd_file(int fd, uoff_t offset, bool autoclose_fd); /* Create an output stream to a buffer. */ struct ostream *o_stream_create_buffer(buffer_t *buf); +/* Create an output streams that always fails the writes. */ +struct ostream *o_stream_create_error(int stream_errno); /* Set name (e.g. path) for output stream. */ void o_stream_set_name(struct ostream *stream, const char *name); From dovecot at dovecot.org Wed Oct 3 02:29:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 02:29:27 +0300 Subject: dovecot-2.2: lib-mail: Moved message_skip_virtual() to message-s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4a074827375d changeset: 15176:4a074827375d user: Timo Sirainen date: Wed Oct 03 02:28:44 2012 +0300 description: lib-mail: Moved message_skip_virtual() to message-size.[ch] and changed API diffstat: src/lib-mail/Makefile.am | 2 - src/lib-mail/message-send.c | 70 --------------------------------------------- src/lib-mail/message-send.h | 14 --------- src/lib-mail/message-size.c | 45 ++++++++++++++++++++++++++++ src/lib-mail/message-size.h | 6 +++ 5 files changed, 51 insertions(+), 86 deletions(-) diffs (177 lines): diff -r 5cdd5bf63c51 -r 4a074827375d src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Wed Oct 03 02:28:31 2012 +0300 +++ b/src/lib-mail/Makefile.am Wed Oct 03 02:28:44 2012 +0300 @@ -26,7 +26,6 @@ message-parser.c \ message-part-serialize.c \ message-search.c \ - message-send.c \ message-size.c \ quoted-printable.c \ rfc2231-parser.c \ @@ -54,7 +53,6 @@ message-parser.h \ message-part-serialize.h \ message-search.h \ - message-send.h \ message-size.h \ quoted-printable.h \ rfc2231-parser.h \ diff -r 5cdd5bf63c51 -r 4a074827375d src/lib-mail/message-send.c --- a/src/lib-mail/message-send.c Wed Oct 03 02:28:31 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,70 +0,0 @@ -/* Copyright (c) 2002-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "istream.h" -#include "ostream.h" -#include "message-parser.h" -#include "message-send.h" -#include "message-size.h" - -int message_skip_virtual(struct istream *input, uoff_t virtual_skip, - struct message_size *msg_size, - bool cr_skipped, bool *last_cr) -{ - const unsigned char *msg; - size_t i, size, startpos; - int ret; - - if (virtual_skip == 0) { - *last_cr = cr_skipped; - return 0; - } - - *last_cr = FALSE; - startpos = 0; - while ((ret = i_stream_read_data(input, &msg, &size, startpos)) > 0) { - for (i = startpos; i < size && virtual_skip > 0; i++) { - virtual_skip--; - - if (msg[i] == '\r') { - /* CR */ - if (virtual_skip == 0) - *last_cr = TRUE; - } else if (msg[i] == '\n') { - /* LF */ - if ((i == 0 && !cr_skipped) || - (i > 0 && msg[i-1] != '\r')) { - /* missing CR */ - if (msg_size != NULL) - msg_size->virtual_size++; - - if (virtual_skip == 0) { - /* CR/LF boundary */ - *last_cr = TRUE; - break; - } - - virtual_skip--; - } - - /* increase after making sure we didn't break - at virtual \r */ - if (msg_size != NULL) - msg_size->lines++; - } - } - - i_stream_skip(input, i); - if (msg_size != NULL) { - msg_size->physical_size += i; - msg_size->virtual_size += i; - } - - if (i < size) - return 0; - - cr_skipped = msg[i-1] == '\r'; - } - i_assert(ret == -1); - return input->stream_errno == 0 ? 0 : -1; -} diff -r 5cdd5bf63c51 -r 4a074827375d src/lib-mail/message-send.h --- a/src/lib-mail/message-send.h Wed Oct 03 02:28:31 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -#ifndef MESSAGE_SEND_H -#define MESSAGE_SEND_H - -struct message_size; - -/* Skip number of virtual bytes from putfer. msg_size is updated if it's not - NULL. If cr_skipped is TRUE and first character is \n, it's not treated as - \r\n. last_cr is set to TRUE if last character we skipped was \r, meaning - that next character should be \n and you shouldn't treat it as \r\n. */ -int message_skip_virtual(struct istream *input, uoff_t virtual_skip, - struct message_size *msg_size, - bool cr_skipped, bool *last_cr); - -#endif diff -r 5cdd5bf63c51 -r 4a074827375d src/lib-mail/message-size.c --- a/src/lib-mail/message-size.c Wed Oct 03 02:28:31 2012 +0300 +++ b/src/lib-mail/message-size.c Wed Oct 03 02:28:44 2012 +0300 @@ -121,3 +121,48 @@ dest->physical_size += src->physical_size; dest->lines += src->lines; } + +int message_skip_virtual(struct istream *input, uoff_t virtual_skip, + bool *last_cr_r) +{ + const unsigned char *msg; + size_t i, size; + bool cr_skipped = FALSE; + int ret; + + *last_cr_r = FALSE; + if (virtual_skip == 0) + return 0; + + while ((ret = i_stream_read_data(input, &msg, &size, 0)) > 0) { + for (i = 0; i < size && virtual_skip > 0; i++) { + virtual_skip--; + + if (msg[i] == '\r') { + /* CR */ + if (virtual_skip == 0) + *last_cr_r = TRUE; + } else if (msg[i] == '\n') { + /* LF */ + if ((i == 0 && !cr_skipped) || + (i > 0 && msg[i-1] != '\r')) { + if (virtual_skip == 0) { + /* CR/LF boundary */ + *last_cr_r = TRUE; + break; + } + + virtual_skip--; + } + } + } + i_stream_skip(input, i); + + if (i < size) + return 0; + + cr_skipped = msg[i-1] == '\r'; + } + i_assert(ret == -1); + return input->stream_errno == 0 ? 0 : -1; +} diff -r 5cdd5bf63c51 -r 4a074827375d src/lib-mail/message-size.h --- a/src/lib-mail/message-size.h Wed Oct 03 02:28:31 2012 +0300 +++ b/src/lib-mail/message-size.h Wed Oct 03 02:28:44 2012 +0300 @@ -19,4 +19,10 @@ void message_size_add(struct message_size *dest, const struct message_size *src); +/* Skip number of virtual bytes from buffer. last_cr_r is set to TRUE if the + last character we skipped was '\r', meaning that the next character should + be '\n', which shouldn't be treated as "\r\n". */ +int message_skip_virtual(struct istream *input, uoff_t virtual_skip, + bool *last_cr_r); + #endif From dovecot at dovecot.org Wed Oct 3 02:29:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 02:29:27 +0300 Subject: dovecot-2.2: imap: Removed unnecessary #includes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5cdd5bf63c51 changeset: 15175:5cdd5bf63c51 user: Timo Sirainen date: Wed Oct 03 02:28:31 2012 +0300 description: imap: Removed unnecessary #includes diffstat: src/imap/imap-fetch-body.c | 1 - src/imap/imap-fetch.c | 1 - 2 files changed, 0 insertions(+), 2 deletions(-) diffs (22 lines): diff -r f860cdf156cf -r 5cdd5bf63c51 src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Sun Sep 30 19:11:55 2012 +0300 +++ b/src/imap/imap-fetch-body.c Wed Oct 03 02:28:31 2012 +0300 @@ -8,7 +8,6 @@ #include "ostream.h" #include "istream-header-filter.h" #include "message-parser.h" -#include "message-send.h" #include "mail-storage-private.h" #include "imap-quote.h" #include "imap-parser.h" diff -r f860cdf156cf -r 5cdd5bf63c51 src/imap/imap-fetch.c --- a/src/imap/imap-fetch.c Sun Sep 30 19:11:55 2012 +0300 +++ b/src/imap/imap-fetch.c Wed Oct 03 02:28:31 2012 +0300 @@ -6,7 +6,6 @@ #include "istream.h" #include "ostream.h" #include "str.h" -#include "message-send.h" #include "message-size.h" #include "imap-date.h" #include "imap-utf7.h" From dovecot at dovecot.org Wed Oct 3 02:29:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 02:29:27 +0300 Subject: dovecot-2.2: imap: Fixed partial FETCHes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99843f74422a changeset: 15177:99843f74422a user: Timo Sirainen date: Wed Oct 03 02:29:02 2012 +0300 description: imap: Fixed partial FETCHes. diffstat: src/lib-imap-storage/imap-msgpart.c | 20 ++++++++++---------- 1 files changed, 10 insertions(+), 10 deletions(-) diffs (52 lines): diff -r 4a074827375d -r 99843f74422a src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Wed Oct 03 02:28:44 2012 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Wed Oct 03 02:29:02 2012 +0300 @@ -413,10 +413,9 @@ { struct mail_msgpart_partial_cache *cache = &mail->box->partial_cache; struct istream *crlf_input; - const unsigned char *data; - size_t size; uoff_t physical_start = input->v_offset; uoff_t virtual_skip = msgpart->partial_offset; + bool cr_skipped; i_assert(msgpart->headers == NULL); /* HEADER.FIELDS returns CRLFs */ @@ -433,27 +432,28 @@ message parts. */ skip_using_parts(mail, input, physical_start, &virtual_skip); } - crlf_input = i_stream_create_crlf(input); - i_stream_unref(&input); - input = crlf_input; - i_stream_skip(input, virtual_skip); + if (message_skip_virtual(input, virtual_skip, &cr_skipped) < 0) + return i_stream_create_error(errno); if ((msgpart->partial_offset != 0 || - msgpart->partial_size != (uoff_t)-1) && - i_stream_read_data(input, &data, &size, 0) > 0) { + msgpart->partial_size != (uoff_t)-1) && !input->eof) { /* update cache */ cache->uid = mail->uid; cache->physical_start = physical_start; cache->physical_pos = input->v_offset; cache->virtual_pos = msgpart->partial_offset; - if (data[0] == '\n') { + if (cr_skipped) { /* the physical_pos points to virtual CRLF, but virtual_pos already skipped CR. that can't work, so seek back the virtual CR */ cache->virtual_pos--; } } - return input; + crlf_input = i_stream_create_crlf(input); + if (cr_skipped) + i_stream_skip(crlf_input, 1); + i_stream_unref(&input); + return crlf_input; } static void From dovecot at dovecot.org Wed Oct 3 03:21:08 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 03:21:08 +0300 Subject: dovecot-2.2: imap: Removed unnecessary code. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ab2abb987619 changeset: 15178:ab2abb987619 user: Timo Sirainen date: Wed Oct 03 03:19:20 2012 +0300 description: imap: Removed unnecessary code. diffstat: src/imap/cmd-expunge.c | 1 - src/imap/imap-client.h | 1 - src/imap/imap-sync.c | 1 - 3 files changed, 0 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 99843f74422a -r ab2abb987619 src/imap/cmd-expunge.c --- a/src/imap/cmd-expunge.c Wed Oct 03 02:29:02 2012 +0300 +++ b/src/imap/cmd-expunge.c Wed Oct 03 03:19:20 2012 +0300 @@ -37,7 +37,6 @@ } client->sync_seen_deletes = FALSE; - client->sync_seen_expunges = FALSE; if ((client->enabled_features & MAILBOX_FEATURE_QRESYNC) != 0) { return cmd_sync(cmd, MAILBOX_SYNC_FLAG_EXPUNGE, IMAP_SYNC_FLAG_SAFE, "OK Expunge completed."); diff -r 99843f74422a -r ab2abb987619 src/imap/imap-client.h --- a/src/imap/imap-client.h Wed Oct 03 02:29:02 2012 +0300 +++ b/src/imap/imap-client.h Wed Oct 03 03:19:20 2012 +0300 @@ -150,7 +150,6 @@ /* syncing marks this TRUE when it sees \Deleted flags. this is by EXPUNGE for Outlook-workaround. */ unsigned int sync_seen_deletes:1; - unsigned int sync_seen_expunges:1; unsigned int disconnected:1; unsigned int destroyed:1; unsigned int handling_input:1; diff -r 99843f74422a -r ab2abb987619 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Wed Oct 03 02:29:02 2012 +0300 +++ b/src/imap/imap-sync.c Wed Oct 03 03:19:20 2012 +0300 @@ -574,7 +574,6 @@ } break; case MAILBOX_SYNC_TYPE_EXPUNGE: - ctx->client->sync_seen_expunges = TRUE; ret = imap_sync_send_expunges(ctx, str); if (ret > 0) { /* update only after we're finished, so that From dovecot at dovecot.org Wed Oct 3 03:21:08 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 03:21:08 +0300 Subject: dovecot-2.2: acl: Don't silently ignore permission error on expu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3f55c6ca06f8 changeset: 15179:3f55c6ca06f8 user: Timo Sirainen date: Wed Oct 03 03:20:15 2012 +0300 description: acl: Don't silently ignore permission error on expunge. diffstat: src/plugins/acl/acl-mailbox.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r ab2abb987619 -r 3f55c6ca06f8 src/plugins/acl/acl-mailbox.c --- a/src/plugins/acl/acl-mailbox.c Wed Oct 03 03:19:20 2012 +0300 +++ b/src/plugins/acl/acl-mailbox.c Wed Oct 03 03:20:15 2012 +0300 @@ -319,8 +319,7 @@ /* if we don't have permission, silently return success so users won't see annoying error messages in case their clients try automatic expunging. */ - if (ret < 0) - acl_transaction_set_failure(_mail->transaction); + acl_transaction_set_failure(_mail->transaction); return; } From dovecot at dovecot.org Wed Oct 3 03:21:08 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 03:21:08 +0300 Subject: dovecot-2.2: imap: If CLOSE/EXPUNGE fails with "permission denie... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d40f26407aad changeset: 15180:d40f26407aad user: Timo Sirainen date: Wed Oct 03 03:20:57 2012 +0300 description: imap: If CLOSE/EXPUNGE fails with "permission denied", return tagged OK anyway. diffstat: src/imap/cmd-close.c | 15 ++++++++++++--- src/imap/cmd-expunge.c | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diffs (69 lines): diff -r 3f55c6ca06f8 -r d40f26407aad src/imap/cmd-close.c --- a/src/imap/cmd-close.c Wed Oct 03 03:20:15 2012 +0300 +++ b/src/imap/cmd-close.c Wed Oct 03 03:20:57 2012 +0300 @@ -9,6 +9,8 @@ struct client *client = cmd->client; struct mailbox *mailbox = client->mailbox; struct mail_storage *storage; + const char *errstr, *tagged_reply = "OK Close completed."; + enum mail_error error = MAIL_ERROR_NONE; if (!client_verify_open_mailbox(cmd)) return TRUE; @@ -17,14 +19,21 @@ client->mailbox = NULL; storage = mailbox_get_storage(mailbox); - if (imap_expunge(mailbox, NULL) < 0) - client_send_untagged_storage_error(client, storage); + if (imap_expunge(mailbox, NULL) < 0) { + errstr = mailbox_get_last_error(mailbox, &error); + if (error != MAIL_ERROR_PERM) + client_send_untagged_storage_error(client, storage); + else { + tagged_reply = t_strdup_printf( + "OK Closed without expunging: %s", errstr); + } + } if (mailbox_sync(mailbox, 0) < 0) client_send_untagged_storage_error(client, storage); mailbox_free(&mailbox); client_update_mailbox_flags(client, NULL); - client_send_tagline(cmd, "OK Close completed."); + client_send_tagline(cmd, tagged_reply); return TRUE; } diff -r 3f55c6ca06f8 -r d40f26407aad src/imap/cmd-expunge.c --- a/src/imap/cmd-expunge.c Wed Oct 03 03:20:15 2012 +0300 +++ b/src/imap/cmd-expunge.c Wed Oct 03 03:20:57 2012 +0300 @@ -24,6 +24,8 @@ struct mail_search_args *search_args) { struct client *client = cmd->client; + const char *errstr; + enum mail_error error = MAIL_ERROR_NONE; int ret; ret = imap_expunge(client->mailbox, search_args == NULL ? NULL : @@ -31,9 +33,16 @@ if (search_args != NULL) mail_search_args_unref(&search_args); if (ret < 0) { - client_send_storage_error(cmd, - mailbox_get_storage(client->mailbox)); - return TRUE; + errstr = mailbox_get_last_error(client->mailbox, &error); + if (error != MAIL_ERROR_PERM) { + client_send_storage_error(cmd, + mailbox_get_storage(client->mailbox)); + return TRUE; + } else { + return cmd_sync(cmd, 0, IMAP_SYNC_FLAG_SAFE, + t_strdup_printf("OK Expunge ignored: %s.", + errstr)); + } } client->sync_seen_deletes = FALSE; From dovecot at dovecot.org Wed Oct 3 03:49:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 03:49:05 +0300 Subject: dovecot-2.2: auth: Add and use SCRAM-SHA-1 password scheme Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3e3ac2c16fa4 changeset: 15181:3e3ac2c16fa4 user: Florian Zeitz date: Wed Sep 19 03:13:39 2012 +0200 description: auth: Add and use SCRAM-SHA-1 password scheme diffstat: src/auth/Makefile.am | 1 + src/auth/mech-scram-sha1.c | 177 +++++++++++++++++++++----------------- src/auth/password-scheme-scram.c | 139 ++++++++++++++++++++++++++++++ src/auth/password-scheme.c | 2 + src/auth/password-scheme.h | 6 + 5 files changed, 245 insertions(+), 80 deletions(-) diffs (truncated from 472 to 300 lines): diff -r d40f26407aad -r 3e3ac2c16fa4 src/auth/Makefile.am --- a/src/auth/Makefile.am Wed Oct 03 03:20:57 2012 +0300 +++ b/src/auth/Makefile.am Wed Sep 19 03:13:39 2012 +0200 @@ -44,6 +44,7 @@ password-scheme.c \ password-scheme-crypt.c \ password-scheme-md5crypt.c \ + password-scheme-scram.c \ password-scheme-otp.c \ password-scheme-rpa.c diff -r d40f26407aad -r 3e3ac2c16fa4 src/auth/mech-scram-sha1.c --- a/src/auth/mech-scram-sha1.c Wed Oct 03 03:20:57 2012 +0300 +++ b/src/auth/mech-scram-sha1.c Wed Sep 19 03:13:39 2012 +0200 @@ -1,11 +1,14 @@ /* * SCRAM-SHA-1 SASL authentication, see RFC-5802 * - * Copyright (c) 2011 Florian Zeitz + * Copyright (c) 2011-2012 Florian Zeitz * * This software is released under the MIT license. */ +#include +#include + #include "auth-common.h" #include "base64.h" #include "buffer.h" @@ -15,6 +18,7 @@ #include "safe-memset.h" #include "str.h" #include "strfuncs.h" +#include "strnum.h" #include "mech.h" /* SCRAM hash iteration count. RFC says it SHOULD be at least 4096 */ @@ -29,45 +33,22 @@ /* sent: */ const char *server_first_message; - unsigned char salt[16]; - unsigned char salted_password[SHA1_RESULTLEN]; + const char *snonce; /* received: */ const char *gs2_cbind_flag; const char *cnonce; - const char *snonce; const char *client_first_message_bare; const char *client_final_message_without_proof; buffer_t *proof; + + /* stored */ + buffer_t *stored_key; + buffer_t *server_key; }; -static void Hi(const unsigned char *str, size_t str_size, - const unsigned char *salt, size_t salt_size, unsigned int i, - unsigned char result[SHA1_RESULTLEN]) -{ - struct hmac_context ctx; - unsigned char U[SHA1_RESULTLEN]; - unsigned int j, k; - - /* Calculate U1 */ - hmac_init(&ctx, str, str_size, &hash_method_sha1); - hmac_update(&ctx, salt, salt_size); - hmac_update(&ctx, "\0\0\0\1", 4); - hmac_final(&ctx, U); - - memcpy(result, U, SHA1_RESULTLEN); - - /* Calculate U2 to Ui and Hi */ - for (j = 2; j <= i; j++) { - hmac_init(&ctx, str, str_size, &hash_method_sha1); - hmac_update(&ctx, U, sizeof(U)); - hmac_final(&ctx, U); - for (k = 0; k < SHA1_RESULTLEN; k++) - result[k] ^= U[k]; - } -} - -static const char *get_scram_server_first(struct scram_auth_request *request) +static const char *get_scram_server_first(struct scram_auth_request *request, + int iter, const char *salt) { unsigned char snonce[SCRAM_SERVER_NONCE_LEN+1]; string_t *str; @@ -84,12 +65,9 @@ snonce[sizeof(snonce)-1] = '\0'; request->snonce = p_strndup(request->pool, snonce, sizeof(snonce)); - random_fill(request->salt, sizeof(request->salt)); - - str = t_str_new(MAX_BASE64_ENCODED_SIZE(sizeof(request->salt))); - str_printfa(str, "r=%s%s,s=", request->cnonce, request->snonce); - base64_encode(request->salt, sizeof(request->salt), str); - str_printfa(str, ",i=%d", SCRAM_ITERATE_COUNT); + str = t_str_new(sizeof(snonce)); + str_printfa(str, "r=%s%s,s=%s,i=%d", request->cnonce, request->snonce, + salt, iter); return str_c(str); } @@ -105,15 +83,8 @@ request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_init(&ctx, request->salted_password, - sizeof(request->salted_password), &hash_method_sha1); - hmac_update(&ctx, "Server Key", 10); - hmac_final(&ctx, server_key); - - safe_memset(request->salted_password, 0, - sizeof(request->salted_password)); - - hmac_init(&ctx, server_key, sizeof(server_key), &hash_method_sha1); + hmac_init(&ctx, request->server_key->data, request->server_key->used, + &hash_method_sha1); hmac_update(&ctx, auth_message, strlen(auth_message)); hmac_final(&ctx, server_signature); @@ -211,8 +182,7 @@ return TRUE; } -static bool verify_credentials(struct scram_auth_request *request, - const unsigned char *credentials, size_t size) +static bool verify_credentials(struct scram_auth_request *request) { struct hmac_context ctx; const char *auth_message; @@ -221,54 +191,90 @@ unsigned char stored_key[SHA1_RESULTLEN]; size_t i; - /* FIXME: credentials should be SASLprepped UTF8 data here */ - Hi(credentials, size, request->salt, sizeof(request->salt), - SCRAM_ITERATE_COUNT, request->salted_password); - - hmac_init(&ctx, request->salted_password, - sizeof(request->salted_password), &hash_method_sha1); - hmac_update(&ctx, "Client Key", 10); - hmac_final(&ctx, client_key); - - sha1_get_digest(client_key, sizeof(client_key), stored_key); - auth_message = t_strconcat(request->client_first_message_bare, ",", request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_init(&ctx, stored_key, sizeof(stored_key), &hash_method_sha1); + hmac_init(&ctx, request->stored_key->data, request->stored_key->used, + &hash_method_sha1); hmac_update(&ctx, auth_message, strlen(auth_message)); hmac_final(&ctx, client_signature); for (i = 0; i < sizeof(client_signature); i++) - client_signature[i] ^= client_key[i]; + client_key[i] = + ((char*)request->proof->data)[i] ^ client_signature[i]; + + sha1_get_digest(client_key, sizeof(client_key), stored_key); safe_memset(client_key, 0, sizeof(client_key)); - safe_memset(stored_key, 0, sizeof(stored_key)); + safe_memset(client_signature, 0, sizeof(client_signature)); - return memcmp(client_signature, request->proof->data, - request->proof->used) == 0; + return memcmp(stored_key, request->stored_key->data, + request->stored_key->used) == 0; } static void credentials_callback(enum passdb_result result, const unsigned char *credentials, size_t size, struct auth_request *auth_request) { + const char *const *fields; + size_t len; + unsigned int iter; + const char *salt; struct scram_auth_request *request = (struct scram_auth_request *)auth_request; - const char *server_final_message; switch (result) { case PASSDB_RESULT_OK: - if (!verify_credentials(request, credentials, size)) { + fields = t_strsplit(t_strndup(credentials, size), ","); + + if (str_array_length(fields) != 4) { auth_request_log_info(auth_request, "scram-sha-1", - "password mismatch"); + "Invalid passdb entry"); auth_request_fail(auth_request); - } else { - server_final_message = get_scram_server_final(request); - auth_request_success(auth_request, server_final_message, - strlen(server_final_message)); + break; } + + if (str_to_uint(fields[0], &iter) < 0 || (iter < 4096) || + (iter > INT_MAX)) { + auth_request_log_info(auth_request, "scram-sha-1", + "Invalid iteration count"); + auth_request_fail(auth_request); + break; + } + + salt = fields[1]; + + len = strlen(fields[2]); + request->stored_key = buffer_create_dynamic(request->pool, + MAX_BASE64_DECODED_SIZE(len)); + if (base64_decode(fields[2], len, NULL, + request->stored_key) < 0) { + auth_request_log_info(auth_request, "scram-sha-1", + "Invalid base64 encoding" + "of StoredKey in passdb"); + auth_request_fail(auth_request); + break; + } + + len = strlen(fields[3]); + request->server_key = buffer_create_dynamic(request->pool, + MAX_BASE64_DECODED_SIZE(len)); + if (base64_decode(fields[3], len, NULL, + request->server_key) < 0) { + auth_request_log_info(auth_request, "scram-sha-1", + "Invalid base64 encoding" + "of ServerKey in passdb"); + auth_request_fail(auth_request); + break; + } + + request->server_first_message = p_strdup(request->pool, + get_scram_server_first(request, iter, salt)); + + auth_request_handler_reply_continue(auth_request, + request->server_first_message, + strlen(request->server_first_message)); break; case PASSDB_RESULT_INTERNAL_FAILURE: auth_request_internal_failure(auth_request); @@ -333,8 +339,6 @@ request->client_final_message_without_proof = p_strdup(request->pool, t_strarray_join(fields, ",")); - auth_request_lookup_credentials(&request->auth_request, "PLAIN", - credentials_callback); return TRUE; } @@ -345,22 +349,35 @@ struct scram_auth_request *request = (struct scram_auth_request *)auth_request; const char *error = NULL; + const char *server_final_message; + int len; if (!request->client_first_message_bare) { /* Received client-first-message */ if (parse_scram_client_first(request, data, data_size, &error)) { - request->server_first_message = p_strdup(request->pool, - get_scram_server_first(request)); - auth_request_handler_reply_continue(auth_request, - request->server_first_message, - strlen(request->server_first_message)); + auth_request_lookup_credentials(&request->auth_request, + "SCRAM-SHA-1", + credentials_callback); return; } } else { /* Received client-final-message */ - if (parse_scram_client_final(request, data, data_size, &error)) - return; + if (parse_scram_client_final(request, data, data_size, + &error)) { + if (!verify_credentials(request)) { + auth_request_log_info(auth_request, + "scram-sha-1", + "password mismatch"); + } else { + server_final_message = + get_scram_server_final(request); + len = strlen(server_final_message); + auth_request_success(auth_request, + server_final_message, len); + return; + } + } } From dovecot at dovecot.org Wed Oct 3 04:02:20 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 04:02:20 +0300 Subject: dovecot-2.2: Typofix: s/TRANSCATION/TRANSACTION/ Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee0cf9c67571 changeset: 15182:ee0cf9c67571 user: Timo Sirainen date: Wed Oct 03 04:01:59 2012 +0300 description: Typofix: s/TRANSCATION/TRANSACTION/ diffstat: src/lib-index/mail-transaction-log-file.c | 4 ++-- src/lib-index/mail-transaction-log-private.h | 4 ++-- src/lib-index/mail-transaction-log.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diffs (50 lines): diff -r 3e3ac2c16fa4 -r ee0cf9c67571 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Wed Sep 19 03:13:39 2012 +0200 +++ b/src/lib-index/mail-transaction-log-file.c Wed Oct 03 04:01:59 2012 +0300 @@ -349,7 +349,7 @@ return mail_transaction_log_file_dotlock(file); i_assert(file->file_lock == NULL); - lock_timeout_secs = I_MIN(MAIL_TRANSCATION_LOG_LOCK_TIMEOUT, + lock_timeout_secs = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT, file->log->index->max_lock_timeout_secs); ret = mail_index_lock_fd(file->log->index, file->filepath, file->fd, F_WRLCK, lock_timeout_secs, @@ -386,7 +386,7 @@ return; lock_time = time(NULL) - file->lock_created; - if (lock_time >= MAIL_TRANSCATION_LOG_LOCK_TIMEOUT) { + if (lock_time >= MAIL_TRANSACTION_LOG_LOCK_TIMEOUT) { i_warning("Transaction log file %s was locked for %u seconds", file->filepath, lock_time); } diff -r 3e3ac2c16fa4 -r ee0cf9c67571 src/lib-index/mail-transaction-log-private.h --- a/src/lib-index/mail-transaction-log-private.h Wed Sep 19 03:13:39 2012 +0200 +++ b/src/lib-index/mail-transaction-log-private.h Wed Oct 03 04:01:59 2012 +0300 @@ -8,8 +8,8 @@ /* Synchronization can take a while sometimes, especially when copying lots of mails. */ -#define MAIL_TRANSCATION_LOG_LOCK_TIMEOUT (3*60) -#define MAIL_TRANSCATION_LOG_LOCK_CHANGE_TIMEOUT (3*60) +#define MAIL_TRANSACTION_LOG_LOCK_TIMEOUT (3*60) +#define MAIL_TRANSACTION_LOG_LOCK_CHANGE_TIMEOUT (3*60) /* Rotate when log is older than ROTATE_TIME and larger than MIN_SIZE */ #define MAIL_TRANSACTION_LOG_ROTATE_MIN_SIZE (1024*32) diff -r 3e3ac2c16fa4 -r ee0cf9c67571 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Wed Sep 19 03:13:39 2012 +0200 +++ b/src/lib-index/mail-transaction-log.c Wed Oct 03 04:01:59 2012 +0300 @@ -556,9 +556,9 @@ struct mail_index *index = log->index; memset(set_r, 0, sizeof(*set_r)); - set_r->timeout = I_MIN(MAIL_TRANSCATION_LOG_LOCK_TIMEOUT, + set_r->timeout = I_MIN(MAIL_TRANSACTION_LOG_LOCK_TIMEOUT, index->max_lock_timeout_secs); - set_r->stale_timeout = MAIL_TRANSCATION_LOG_LOCK_CHANGE_TIMEOUT; + set_r->stale_timeout = MAIL_TRANSACTION_LOG_LOCK_CHANGE_TIMEOUT; set_r->nfs_flush = (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0; set_r->use_excl_lock = (index->flags & MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL) != 0; From dovecot at dovecot.org Wed Oct 3 04:09:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 04:09:33 +0300 Subject: dovecot-2.2: lib-master: Fixed crashing with -i param... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04cd35e86025 changeset: 15183:04cd35e86025 user: Timo Sirainen date: Wed Oct 03 04:09:23 2012 +0300 description: lib-master: Fixed crashing with -i parameter handling. diffstat: src/lib-master/master-service.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r ee0cf9c67571 -r 04cd35e86025 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Wed Oct 03 04:01:59 2012 +0300 +++ b/src/lib-master/master-service.c Wed Oct 03 04:09:23 2012 +0300 @@ -336,8 +336,10 @@ const struct master_instance *inst; const char *instance_path, *path; - instance_path = t_strconcat(master_service->set->state_dir, - "/"MASTER_INSTANCE_FNAME, NULL); + /* note that we don't have any settings yet. we're just finding out + which dovecot.conf we even want to read! so we must use the + hardcoded state_dir path. */ + instance_path = t_strconcat(PKG_STATEDIR"/"MASTER_INSTANCE_FNAME, NULL); list = master_instance_list_init(instance_path); inst = master_instance_list_find_by_name(list, name); if (inst != NULL) { From dovecot at dovecot.org Wed Oct 3 04:14:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 04:14:44 +0300 Subject: dovecot-2.2: lib-dict: Fixed error handling on initialization. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9f86c18e6b2c changeset: 15184:9f86c18e6b2c user: Timo Sirainen date: Wed Oct 03 04:14:33 2012 +0300 description: lib-dict: Fixed error handling on initialization. diffstat: src/lib-dict/dict-file.c | 6 ++- src/lib-dict/dict-memcached-ascii.c | 66 +++++++++++++++++++++------------- src/lib-dict/dict-memcached.c | 21 ++++++++-- src/lib-dict/dict-redis.c | 70 ++++++++++++++++++++++-------------- 4 files changed, 104 insertions(+), 59 deletions(-) diffs (256 lines): diff -r 04cd35e86025 -r 9f86c18e6b2c src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Wed Oct 03 04:09:23 2012 +0300 +++ b/src/lib-dict/dict-file.c Wed Oct 03 04:14:33 2012 +0300 @@ -73,8 +73,12 @@ dict->lock_method = FILE_LOCK_METHOD_FCNTL; else if (strcmp(p, "lock=flock") == 0) dict->lock_method = FILE_LOCK_METHOD_FLOCK; - else + else { i_error("dict file: Invalid parameter: %s", p+1); + i_free(dict->path); + i_free(dict); + return -1; + } } dict->dict = *driver; dict->hash_pool = pool_alloconly_create("file dict", 1024); diff -r 04cd35e86025 -r 9f86c18e6b2c src/lib-dict/dict-memcached-ascii.c --- a/src/lib-dict/dict-memcached-ascii.c Wed Oct 03 04:09:23 2012 +0300 +++ b/src/lib-dict/dict-memcached-ascii.c Wed Oct 03 04:14:33 2012 +0300 @@ -340,6 +340,7 @@ struct memcached_ascii_dict *dict; const char *const *args; struct ioloop *old_ioloop = current_ioloop; + int ret = 0; if (memcached_ascii_connections == NULL) { memcached_ascii_connections = @@ -352,41 +353,54 @@ i_unreached(); dict->port = MEMCACHED_DEFAULT_PORT; dict->timeout_msecs = MEMCACHED_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (strncmp(*args, "host=", 5) == 0) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + i_error("Invalid IP: %s", *args+5); + ret = -1; + } + } else if (strncmp(*args, "port=", 5) == 0) { + if (str_to_uint(*args+5, &dict->port) < 0) { + i_error("Invalid port: %s", *args+5); + ret = -1; + } + } else if (strncmp(*args, "prefix=", 7) == 0) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + i_error("Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + i_error("Unknown parameter: %s", *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + + connection_init_client_ip(memcached_ascii_connections, &dict->conn.conn, + &dict->ip, dict->port); + dict->dict = *driver; + dict->conn.reply_str = str_new(default_pool, 256); + dict->conn.dict = dict; + if (strchr(username, DICT_USERNAME_SEPARATOR) == NULL) dict->username = i_strdup(username); else { /* escape the username */ dict->username = i_strdup(memcached_ascii_escape_username(username)); } - dict->key_prefix = i_strdup(""); i_array_init(&dict->input_states, 4); i_array_init(&dict->replies, 4); - args = t_strsplit(uri, ":"); - for (; *args != NULL; args++) { - if (strncmp(*args, "host=", 5) == 0) { - if (net_addr2ip(*args+5, &dict->ip) < 0) - i_error("Invalid IP: %s", *args+5); - } else if (strncmp(*args, "port=", 5) == 0) { - if (str_to_uint(*args+5, &dict->port) < 0) - i_error("Invalid port: %s", *args+5); - } else if (strncmp(*args, "prefix=", 7) == 0) { - i_free(dict->key_prefix); - dict->key_prefix = i_strdup(*args + 7); - } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { - if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) - i_error("Invalid timeout_msecs: %s", *args+14); - } else { - i_error("Unknown parameter: %s", *args); - } - } - connection_init_client_ip(memcached_ascii_connections, &dict->conn.conn, - &dict->ip, dict->port); - - dict->dict = *driver; - dict->conn.reply_str = str_new(default_pool, 256); - dict->conn.dict = dict; - dict->ioloop = io_loop_create(); current_ioloop = old_ioloop; *dict_r = &dict->dict; diff -r 04cd35e86025 -r 9f86c18e6b2c src/lib-dict/dict-memcached.c --- a/src/lib-dict/dict-memcached.c Wed Oct 03 04:09:23 2012 +0300 +++ b/src/lib-dict/dict-memcached.c Wed Oct 03 04:14:33 2012 +0300 @@ -175,6 +175,7 @@ { struct memcached_dict *dict; const char *const *args; + int ret = 0; if (memcached_connections == NULL) { memcached_connections = @@ -192,24 +193,36 @@ args = t_strsplit(uri, ":"); for (; *args != NULL; args++) { if (strncmp(*args, "host=", 5) == 0) { - if (net_addr2ip(*args+5, &dict->ip) < 0) + if (net_addr2ip(*args+5, &dict->ip) < 0) { i_error("Invalid IP: %s", *args+5); + ret = -1; + } } else if (strncmp(*args, "port=", 5) == 0) { - if (str_to_uint(*args+5, &dict->port) < 0) + if (str_to_uint(*args+5, &dict->port) < 0) { i_error("Invalid port: %s", *args+5); + ret = -1; + } } else if (strncmp(*args, "prefix=", 7) == 0) { i_free(dict->key_prefix); dict->key_prefix = i_strdup(*args + 7); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { - if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { i_error("Invalid timeout_msecs: %s", *args+14); + ret = -1; + } } else { i_error("Unknown parameter: %s", *args); + ret = -1; } } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + connection_init_client_ip(memcached_connections, &dict->conn.conn, &dict->ip, dict->port); - dict->dict = *driver; dict->conn.cmd = buffer_create_dynamic(default_pool, 256); dict->conn.dict = dict; diff -r 04cd35e86025 -r 9f86c18e6b2c src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Wed Oct 03 04:09:23 2012 +0300 +++ b/src/lib-dict/dict-redis.c Wed Oct 03 04:14:33 2012 +0300 @@ -306,6 +306,7 @@ { struct redis_dict *dict; const char *const *args; + int ret = 0; if (redis_connections == NULL) { redis_connections = @@ -318,42 +319,55 @@ i_unreached(); dict->port = REDIS_DEFAULT_PORT; dict->timeout_msecs = REDIS_DEFAULT_LOOKUP_TIMEOUT_MSECS; + dict->key_prefix = i_strdup(""); + + args = t_strsplit(uri, ":"); + for (; *args != NULL; args++) { + if (strncmp(*args, "host=", 5) == 0) { + if (net_addr2ip(*args+5, &dict->ip) < 0) { + i_error("Invalid IP: %s", *args+5); + ret = -1; + } + } else if (strncmp(*args, "port=", 5) == 0) { + if (str_to_uint(*args+5, &dict->port) < 0) { + i_error("Invalid port: %s", *args+5); + ret = -1; + } + } else if (strncmp(*args, "prefix=", 7) == 0) { + i_free(dict->key_prefix); + dict->key_prefix = i_strdup(*args + 7); + } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { + if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { + i_error("Invalid timeout_msecs: %s", *args+14); + ret = -1; + } + } else { + i_error("Unknown parameter: %s", *args); + ret = -1; + } + } + if (ret < 0) { + i_free(dict->key_prefix); + i_free(dict); + return -1; + } + connection_init_client_ip(redis_connections, &dict->conn.conn, + &dict->ip, dict->port); + dict->dict = *driver; + dict->conn.last_reply = str_new(default_pool, 256); + dict->conn.dict = dict; + + i_array_init(&dict->input_states, 4); + i_array_init(&dict->replies, 4); if (strchr(username, DICT_USERNAME_SEPARATOR) == NULL) dict->username = i_strdup(username); else { /* escape the username */ dict->username = i_strdup(redis_escape_username(username)); } - dict->key_prefix = i_strdup(""); - i_array_init(&dict->input_states, 4); - i_array_init(&dict->replies, 4); - args = t_strsplit(uri, ":"); - for (; *args != NULL; args++) { - if (strncmp(*args, "host=", 5) == 0) { - if (net_addr2ip(*args+5, &dict->ip) < 0) - i_error("Invalid IP: %s", *args+5); - } else if (strncmp(*args, "port=", 5) == 0) { - if (str_to_uint(*args+5, &dict->port) < 0) - i_error("Invalid port: %s", *args+5); - } else if (strncmp(*args, "prefix=", 7) == 0) { - i_free(dict->key_prefix); - dict->key_prefix = i_strdup(*args + 7); - } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { - if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) - i_error("Invalid timeout_msecs: %s", *args+14); - } else { - i_error("Unknown parameter: %s", *args); - } - } - connection_init_client_ip(redis_connections, &dict->conn.conn, - &dict->ip, dict->port); - - dict->dict = *driver; - dict->conn.last_reply = str_new(default_pool, 256); - dict->conn.dict = dict; *dict_r = &dict->dict; - return -1; + return 0; } static void redis_dict_deinit(struct dict *_dict) From dovecot at dovecot.org Wed Oct 3 05:15:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 05:15:23 +0300 Subject: dovecot-2.2: maildir: Fixed crash with mailbox_list_index=yes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/62a14c9ae6c4 changeset: 15185:62a14c9ae6c4 user: Timo Sirainen date: Wed Oct 03 05:15:11 2012 +0300 description: maildir: Fixed crash with mailbox_list_index=yes diffstat: src/lib-storage/index/maildir/maildir-sync-index.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (16 lines): diff -r 9f86c18e6b2c -r 62a14c9ae6c4 src/lib-storage/index/maildir/maildir-sync-index.c --- a/src/lib-storage/index/maildir/maildir-sync-index.c Wed Oct 03 04:14:33 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Wed Oct 03 05:15:11 2012 +0300 @@ -728,7 +728,11 @@ return 1; } - root_dir = mailbox_get_path(box); + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &root_dir); + if (ret < 0) + return ret; + i_assert(ret > 0); /* check if new/ changed */ new_dir = t_strconcat(root_dir, "/new", NULL); From dovecot at dovecot.org Wed Oct 3 05:20:41 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 05:20:41 +0300 Subject: dovecot-2.1: imap: LIST (SPECIAL-USE) shouldn't send INBOX reply. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/12606087a774 changeset: 14751:12606087a774 user: Timo Sirainen date: Wed Oct 03 05:20:29 2012 +0300 description: imap: LIST (SPECIAL-USE) shouldn't send INBOX reply. diffstat: src/imap/cmd-list.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r b99bead8b339 -r 12606087a774 src/imap/cmd-list.c --- a/src/imap/cmd-list.c Wed Oct 03 01:38:20 2012 +0300 +++ b/src/imap/cmd-list.c Wed Oct 03 05:20:29 2012 +0300 @@ -878,7 +878,8 @@ /* INBOX always exists */ if (!ctx->inbox_found && ctx->cur_ns_match_inbox && (ctx->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && - (ctx->list_flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0) { + (ctx->list_flags & (MAILBOX_LIST_ITER_SELECT_SUBSCRIBED | + MAILBOX_LIST_ITER_SELECT_SPECIALUSE)) == 0) { str = t_str_new(64); str_append(str, "* LIST (\\Unmarked) "); list_reply_append_ns_sep_param(str, From dovecot at dovecot.org Wed Oct 3 05:43:35 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 05:43:35 +0300 Subject: dovecot-2.1: lib-storage: When configuring mailbox INBOX {}, mak... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/16f2f71c91b1 changeset: 14753:16f2f71c91b1 user: Timo Sirainen date: Wed Oct 03 05:42:55 2012 +0300 description: lib-storage: When configuring mailbox INBOX {}, make sure INBOX is uppercased. diffstat: src/lib-storage/mailbox-list-iter.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 4308d1794328 -r 16f2f71c91b1 src/lib-storage/mailbox-list-iter.c --- a/src/lib-storage/mailbox-list-iter.c Wed Oct 03 05:41:46 2012 +0300 +++ b/src/lib-storage/mailbox-list-iter.c Wed Oct 03 05:42:55 2012 +0300 @@ -137,6 +137,11 @@ autobox = array_append_space(&actx->boxes); autobox->name = set->name; autobox->set = set; + if (strcasecmp(autobox->name, "INBOX") == 0) { + /* make sure duplicate INBOX/Inbox/etc. + won't get created */ + autobox->name = "INBOX"; + } } } } From dovecot at dovecot.org Wed Oct 3 05:43:35 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 05:43:35 +0300 Subject: dovecot-2.1: lib-index: Fix for handling view syncing for alread... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/4308d1794328 changeset: 14752:4308d1794328 user: Timo Sirainen date: Wed Oct 03 05:41:46 2012 +0300 description: lib-index: Fix for handling view syncing for already deleted transaction logs. The sync changes' hidden-flag was set randomly, which could have caused flag changes to get lost. diffstat: src/lib-index/mail-index-view-sync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 12606087a774 -r 4308d1794328 src/lib-index/mail-index-view-sync.c --- a/src/lib-index/mail-index-view-sync.c Wed Oct 03 05:20:29 2012 +0300 +++ b/src/lib-index/mail-index-view-sync.c Wed Oct 03 05:41:46 2012 +0300 @@ -767,7 +767,7 @@ /* skip internal flag changes */ if (ctx->data_offset == ctx->hdr->size) - return 0; + return FALSE; update = CONST_PTR_OFFSET(data, ctx->data_offset); } @@ -835,6 +835,7 @@ sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; sync_rec->uid1 = range[ctx->lost_flag_idx].seq1; sync_rec->uid2 = range[ctx->lost_flag_idx].seq2; + sync_rec->hidden = FALSE; ctx->lost_flag_idx++; return TRUE; } From dovecot at dovecot.org Wed Oct 3 05:43:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 05:43:36 +0300 Subject: dovecot-2.1: lib-storage: Fixed potential memory leak when mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/28be06dbd65e changeset: 14754:28be06dbd65e user: Timo Sirainen date: Wed Oct 03 05:43:27 2012 +0300 description: lib-storage: Fixed potential memory leak when mailbox_transaction_commit_get_changes() failed diffstat: src/lib-storage/mail-storage.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 16f2f71c91b1 -r 28be06dbd65e src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Wed Oct 03 05:42:55 2012 +0300 +++ b/src/lib-storage/mail-storage.c Wed Oct 03 05:43:27 2012 +0300 @@ -1485,7 +1485,6 @@ /* Store changes temporarily so that plugins overriding transaction_commit() can look at them. */ - changes.pool = NULL; ret = mailbox_transaction_commit_get_changes(t, &changes); if (changes.pool != NULL) pool_unref(&changes.pool); @@ -1500,11 +1499,14 @@ int ret; t->box->transaction_count--; + changes_r->pool = NULL; *_t = NULL; T_BEGIN { ret = t->box->v.transaction_commit(t, changes_r); } T_END; + if (ret < 0 && changes_r->pool != NULL) + pool_unref(&changes_r->pool); return ret; } From dovecot at dovecot.org Wed Oct 3 16:57:43 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 16:57:43 +0300 Subject: dovecot-2.2: auth: More error checking and cleanups to SCRAM-SHA-1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/559718e321a2 changeset: 15186:559718e321a2 user: Timo Sirainen date: Wed Oct 03 16:57:28 2012 +0300 description: auth: More error checking and cleanups to SCRAM-SHA-1. diffstat: src/auth/mech-scram-sha1.c | 65 ++++++------------------------- src/auth/password-scheme-scram.c | 81 +++++++++++++++++++++++++++++++-------- src/auth/password-scheme.h | 4 + 3 files changed, 81 insertions(+), 69 deletions(-) diffs (278 lines): diff -r 62a14c9ae6c4 -r 559718e321a2 src/auth/mech-scram-sha1.c --- a/src/auth/mech-scram-sha1.c Wed Oct 03 05:15:11 2012 +0300 +++ b/src/auth/mech-scram-sha1.c Wed Oct 03 16:57:28 2012 +0300 @@ -19,10 +19,9 @@ #include "str.h" #include "strfuncs.h" #include "strnum.h" +#include "password-scheme.h" #include "mech.h" -/* SCRAM hash iteration count. RFC says it SHOULD be at least 4096 */ -#define SCRAM_ITERATE_COUNT 4096 /* s-nonce length */ #define SCRAM_SERVER_NONCE_LEN 64 @@ -43,8 +42,8 @@ buffer_t *proof; /* stored */ - buffer_t *stored_key; - buffer_t *server_key; + unsigned char stored_key[SHA1_RESULTLEN]; + unsigned char server_key[SHA1_RESULTLEN]; }; static const char *get_scram_server_first(struct scram_auth_request *request, @@ -75,7 +74,6 @@ { struct hmac_context ctx; const char *auth_message; - unsigned char server_key[SHA1_RESULTLEN]; unsigned char server_signature[SHA1_RESULTLEN]; string_t *str; @@ -83,7 +81,7 @@ request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_init(&ctx, request->server_key->data, request->server_key->used, + hmac_init(&ctx, request->server_key, sizeof(request->server_key), &hash_method_sha1); hmac_update(&ctx, auth_message, strlen(auth_message)); hmac_final(&ctx, server_signature); @@ -195,7 +193,7 @@ request->server_first_message, ",", request->client_final_message_without_proof, NULL); - hmac_init(&ctx, request->stored_key->data, request->stored_key->used, + hmac_init(&ctx, request->stored_key, sizeof(request->stored_key), &hash_method_sha1); hmac_update(&ctx, auth_message, strlen(auth_message)); hmac_final(&ctx, client_signature); @@ -209,68 +207,31 @@ safe_memset(client_key, 0, sizeof(client_key)); safe_memset(client_signature, 0, sizeof(client_signature)); - return memcmp(stored_key, request->stored_key->data, - request->stored_key->used) == 0; + return memcmp(stored_key, request->stored_key, sizeof(stored_key)) == 0; } static void credentials_callback(enum passdb_result result, const unsigned char *credentials, size_t size, struct auth_request *auth_request) { - const char *const *fields; - size_t len; - unsigned int iter; - const char *salt; struct scram_auth_request *request = (struct scram_auth_request *)auth_request; + const char *salt, *error; + unsigned int iter_count; switch (result) { case PASSDB_RESULT_OK: - fields = t_strsplit(t_strndup(credentials, size), ","); - - if (str_array_length(fields) != 4) { + if (scram_sha1_scheme_parse(credentials, size, &iter_count, + &salt, request->stored_key, + request->server_key, &error) < 0) { auth_request_log_info(auth_request, "scram-sha-1", - "Invalid passdb entry"); - auth_request_fail(auth_request); - break; - } - - if (str_to_uint(fields[0], &iter) < 0 || (iter < 4096) || - (iter > INT_MAX)) { - auth_request_log_info(auth_request, "scram-sha-1", - "Invalid iteration count"); - auth_request_fail(auth_request); - break; - } - - salt = fields[1]; - - len = strlen(fields[2]); - request->stored_key = buffer_create_dynamic(request->pool, - MAX_BASE64_DECODED_SIZE(len)); - if (base64_decode(fields[2], len, NULL, - request->stored_key) < 0) { - auth_request_log_info(auth_request, "scram-sha-1", - "Invalid base64 encoding" - "of StoredKey in passdb"); - auth_request_fail(auth_request); - break; - } - - len = strlen(fields[3]); - request->server_key = buffer_create_dynamic(request->pool, - MAX_BASE64_DECODED_SIZE(len)); - if (base64_decode(fields[3], len, NULL, - request->server_key) < 0) { - auth_request_log_info(auth_request, "scram-sha-1", - "Invalid base64 encoding" - "of ServerKey in passdb"); + "%s", error); auth_request_fail(auth_request); break; } request->server_first_message = p_strdup(request->pool, - get_scram_server_first(request, iter, salt)); + get_scram_server_first(request, iter_count, salt)); auth_request_handler_reply_continue(auth_request, request->server_first_message, diff -r 62a14c9ae6c4 -r 559718e321a2 src/auth/password-scheme-scram.c --- a/src/auth/password-scheme-scram.c Wed Oct 03 05:15:11 2012 +0300 +++ b/src/auth/password-scheme-scram.c Wed Oct 03 16:57:28 2012 +0300 @@ -18,8 +18,11 @@ #include "str.h" #include "password-scheme.h" -/* SCRAM hash iteration count. RFC says it SHOULD be at least 4096 */ -#define SCRAM_ITERATE_COUNT 4096 +/* SCRAM allowed iteration count range. RFC says it SHOULD be at least 4096 */ +#define SCRAM_MIN_ITERATE_COUNT 4096 +#define SCRAM_MAX_ITERATE_COUNT INT_MAX + +#define SCRAM_DEFAULT_ITERATE_COUNT 4096 static void Hi(const unsigned char *str, size_t str_size, const unsigned char *salt, size_t salt_size, unsigned int i, @@ -47,30 +50,73 @@ } } -/* password string format: iter,salt,stored_key,server_key */ +int scram_sha1_scheme_parse(const unsigned char *credentials, size_t size, + unsigned int *iter_count_r, const char **salt_r, + unsigned char stored_key_r[], + unsigned char server_key_r[], const char **error_r) +{ + const char *const *fields; + buffer_t *buf; + + /* password string format: iter,salt,stored_key,server_key */ + fields = t_strsplit(t_strndup(credentials, size), ","); + + if (str_array_length(fields) != 4) { + *error_r = "Invalid SCRAM-SHA-1 passdb entry format"; + return -1; + } + if (str_to_uint(fields[0], iter_count_r) < 0 || + *iter_count_r < SCRAM_MIN_ITERATE_COUNT || + *iter_count_r > SCRAM_MAX_ITERATE_COUNT) { + *error_r = "Invalid SCRAM-SHA-1 iteration count in passdb"; + return -1; + } + *salt_r = fields[1]; + + buf = buffer_create_dynamic(pool_datastack_create(), SHA1_RESULTLEN); + if (base64_decode(fields[2], strlen(fields[2]), NULL, buf) < 0 || + buf->used != SHA1_RESULTLEN) { + *error_r = "Invalid SCRAM-SHA-1 StoredKey in passdb"; + return -1; + } + memcpy(stored_key_r, buf->data, SHA1_RESULTLEN); + + buffer_set_used_size(buf, 0); + if (base64_decode(fields[3], strlen(fields[3]), NULL, buf) < 0 || + buf->used != SHA1_RESULTLEN) { + *error_r = "Invalid SCRAM-SHA-1 ServerKey in passdb"; + return -1; + } + memcpy(server_key_r, buf->data, SHA1_RESULTLEN); + return 0; +} int scram_sha1_verify(const char *plaintext, const char *user ATTR_UNUSED, const unsigned char *raw_password, size_t size, - const char **error_r ATTR_UNUSED) + const char **error_r) { struct hmac_context ctx; - string_t *str; - const char *const *fields; - int iter; + const char *salt_base64; + unsigned int iter_count; const unsigned char *salt; size_t salt_len; unsigned char salted_password[SHA1_RESULTLEN]; unsigned char client_key[SHA1_RESULTLEN]; unsigned char stored_key[SHA1_RESULTLEN]; + unsigned char calculated_stored_key[SHA1_RESULTLEN]; + unsigned char server_key[SHA1_RESULTLEN]; + int ret; - fields = t_strsplit(t_strndup(raw_password, size), ","); - iter = atoi(fields[0]); - salt = buffer_get_data(t_base64_decode_str(fields[1]), &salt_len); - str = t_str_new(strlen(fields[2])); + if (scram_sha1_scheme_parse(raw_password, size, &iter_count, + &salt_base64, stored_key, + server_key, error_r) < 0) + return -1; + + salt = buffer_get_data(t_base64_decode_str(salt_base64), &salt_len); /* FIXME: credentials should be SASLprepped UTF8 data here */ Hi((const unsigned char *)plaintext, strlen(plaintext), salt, salt_len, - iter, salted_password); + iter_count, salted_password); /* Calculate ClientKey */ hmac_init(&ctx, salted_password, sizeof(salted_password), @@ -79,14 +125,15 @@ hmac_final(&ctx, client_key); /* Calculate StoredKey */ - sha1_get_digest(client_key, sizeof(client_key), stored_key); - base64_encode(stored_key, sizeof(stored_key), str); + sha1_get_digest(client_key, sizeof(client_key), calculated_stored_key); + ret = memcmp(stored_key, calculated_stored_key, + sizeof(stored_key)) == 0 ? 1 : 0; safe_memset(salted_password, 0, sizeof(salted_password)); safe_memset(client_key, 0, sizeof(client_key)); safe_memset(stored_key, 0, sizeof(stored_key)); - return strcmp(fields[2], str_c(str)) == 0 ? 1 : 0; + return ret; } void scram_sha1_generate(const char *plaintext, const char *user ATTR_UNUSED, @@ -103,12 +150,12 @@ random_fill(salt, sizeof(salt)); str = t_str_new(MAX_BASE64_ENCODED_SIZE(sizeof(salt))); - str_printfa(str, "%i,", SCRAM_ITERATE_COUNT); + str_printfa(str, "%d,", SCRAM_DEFAULT_ITERATE_COUNT); base64_encode(salt, sizeof(salt), str); /* FIXME: credentials should be SASLprepped UTF8 data here */ Hi((const unsigned char *)plaintext, strlen(plaintext), salt, - sizeof(salt), SCRAM_ITERATE_COUNT, salted_password); + sizeof(salt), SCRAM_DEFAULT_ITERATE_COUNT, salted_password); /* Calculate ClientKey */ hmac_init(&ctx, salted_password, sizeof(salted_password), diff -r 62a14c9ae6c4 -r 559718e321a2 src/auth/password-scheme.h --- a/src/auth/password-scheme.h Wed Oct 03 05:15:11 2012 +0300 +++ b/src/auth/password-scheme.h Wed Oct 03 16:57:28 2012 +0300 @@ -85,6 +85,10 @@ const unsigned char *raw_password, size_t size, const char **error_r); +int scram_sha1_scheme_parse(const unsigned char *credentials, size_t size, + unsigned int *iter_count_r, const char **salt_r, + unsigned char stored_key_r[], + unsigned char server_key_r[], const char **error_r); int scram_sha1_verify(const char *plaintext, const char *user ATTR_UNUSED, const unsigned char *raw_password, size_t size, const char **error_r ATTR_UNUSED); From dovecot at dovecot.org Wed Oct 3 18:18:09 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Oct 2012 18:18:09 +0300 Subject: dovecot-2.2: Renamed network.[ch] to net.[ch]. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/02451e967a06 changeset: 15187:02451e967a06 user: Timo Sirainen date: Wed Oct 03 18:17:26 2012 +0300 description: Renamed network.[ch] to net.[ch]. The function prefixes already started with net_ instead of network_. And icecap wants to use network.h for other purpose. :) diffstat: src/auth/auth-client-connection.c | 2 +- src/auth/auth-master-connection.c | 2 +- src/auth/auth-penalty.c | 2 +- src/auth/auth-postfix-connection.c | 2 +- src/auth/auth-request-handler.c | 2 +- src/auth/auth-request.h | 2 +- src/auth/auth-worker-client.c | 2 +- src/auth/auth-worker-server.c | 2 +- src/auth/db-ldap.c | 2 +- src/auth/main.c | 2 +- src/auth/passdb-pam.c | 2 +- src/config/config-filter.h | 2 +- src/config/settings-get.pl | 2 +- src/director/auth-connection.c | 2 +- src/director/director-connection.c | 2 +- src/director/director-host.h | 2 +- src/director/director.h | 2 +- src/director/doveadm-connection.c | 2 +- src/director/login-connection.c | 2 +- src/director/mail-host.h | 2 +- src/doveadm/doveadm-director.c | 2 +- src/doveadm/doveadm-kick.c | 2 +- src/doveadm/doveadm-mail-index.c | 2 +- src/doveadm/doveadm-penalty.c | 2 +- src/doveadm/doveadm-stats.c | 2 +- src/doveadm/doveadm-util.c | 2 +- src/doveadm/doveadm-who.c | 2 +- src/doveadm/server-connection.c | 2 +- src/imap-login/client-authenticate.c | 2 +- src/imap-login/client.h | 2 +- src/imap-urlauth/imap-urlauth-client.c | 2 +- src/imap-urlauth/imap-urlauth-login.c | 2 +- src/imap-urlauth/imap-urlauth-worker.c | 2 +- src/imap/cmd-idle.c | 2 +- src/imap/cmd-urlfetch.c | 2 +- src/imap/imap-client.c | 2 +- src/lib-auth/auth-client.h | 2 +- src/lib-auth/auth-master.c | 2 +- src/lib-auth/auth-master.h | 2 +- src/lib-auth/auth-server-connection.c | 2 +- src/lib-dict/dict-client.c | 2 +- src/lib-dns/dns-lookup.c | 2 +- src/lib-imap-client/imapc-connection.c | 2 +- src/lib-imap-storage/imap-msgpart-url.c | 2 +- src/lib-imap-urlauth/imap-urlauth-connection.c | 2 +- src/lib-imap-urlauth/imap-urlauth-fetch.c | 2 +- src/lib-imap/imap-url.c | 2 +- src/lib-imap/test-imap-url.c | 2 +- src/lib-lda/lmtp-client.c | 2 +- src/lib-lda/lmtp-client.h | 2 +- src/lib-master/anvil-client.c | 2 +- src/lib-master/ipc-client.c | 2 +- src/lib-master/ipc-server.c | 2 +- src/lib-master/master-auth.h | 2 +- src/lib-master/master-login-auth.c | 2 +- src/lib-master/master-service-settings.h | 2 +- src/lib-master/master-service.h | 2 +- src/lib-settings/settings-parser.c | 2 +- src/lib-storage/index/pop3c/pop3c-client.c | 2 +- src/lib-storage/mail-storage-service.h | 2 +- src/lib-storage/mail-user.c | 2 +- src/lib/Makefile.am | 4 +- src/lib/connection.c | 2 +- src/lib/connection.h | 2 +- src/lib/failures.c | 2 +- src/lib/fd-close-on-exec.c | 2 +- src/lib/ioloop-notify-inotify.c | 2 +- src/lib/istream-file.c | 2 +- src/lib/net.c | 1023 ++++++++++++++++++++ src/lib/net.h | 152 ++ src/lib/network.c | 1023 -------------------- src/lib/network.h | 152 -- src/lib/ostream-file.c | 2 +- src/lib/sendfile-util.c | 2 +- src/lib/test-network.c | 2 +- src/lib/unix-socket-create.c | 2 +- src/lib/uri-util.c | 2 +- src/lmtp/client.h | 2 +- src/lmtp/lmtp-proxy.h | 2 +- src/login-common/access-lookup.c | 2 +- src/login-common/client-common.h | 2 +- src/login-common/login-proxy-state.c | 2 +- src/login-common/login-proxy.h | 2 +- src/login-common/ssl-proxy-gnutls.c | 2 +- src/login-common/ssl-proxy-openssl.c | 2 +- src/master/master-settings.c | 2 +- src/master/service-listen.c | 2 +- src/master/service.h | 2 +- src/plugins/fts/fts-indexer.c | 2 +- src/plugins/fts/fts-parser-script.c | 2 +- src/plugins/quota/quota.c | 2 +- src/plugins/replication/replication-plugin.c | 2 +- src/plugins/stats/stats-connection.c | 2 +- src/plugins/zlib/doveadm-zlib.c | 2 +- src/pop3-login/client.h | 2 +- src/pop3/pop3-client.c | 2 +- src/replication/aggregator/notify-connection.c | 2 +- src/replication/aggregator/replicator-connection.c | 2 +- src/replication/replicator/doveadm-connection.c | 2 +- src/stats/client-export.c | 2 +- src/stats/mail-stats.h | 2 +- src/util/rawlog.c | 2 +- 102 files changed, 1274 insertions(+), 1274 deletions(-) diffs (truncated from 3551 to 300 lines): diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-client-connection.c --- a/src/auth/auth-client-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-client-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -4,7 +4,7 @@ #include "ioloop.h" #include "istream.h" #include "ostream.h" -#include "network.h" +#include "net.h" #include "hex-binary.h" #include "hostpid.h" #include "llist.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-master-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -10,7 +10,7 @@ #include "hostpid.h" #include "hex-binary.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "ipwd.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-penalty.c --- a/src/auth/auth-penalty.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-penalty.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "crc32.h" #include "master-service.h" #include "anvil-client.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-postfix-connection.c --- a/src/auth/auth-postfix-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-postfix-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "auth-common.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "llist.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-request-handler.c Wed Oct 03 18:17:26 2012 +0300 @@ -6,7 +6,7 @@ #include "aqueue.h" #include "base64.h" #include "hash.h" -#include "network.h" +#include "net.h" #include "str.h" #include "str-sanitize.h" #include "master-interface.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-request.h --- a/src/auth/auth-request.h Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-request.h Wed Oct 03 18:17:26 2012 +0300 @@ -1,7 +1,7 @@ #ifndef AUTH_REQUEST_H #define AUTH_REQUEST_H -#include "network.h" +#include "net.h" #include "var-expand.h" #include "mech.h" #include "userdb.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-worker-client.c Wed Oct 03 18:17:26 2012 +0300 @@ -3,7 +3,7 @@ #include "auth-common.h" #include "base64.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "hex-binary.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/auth-worker-server.c Wed Oct 03 18:17:26 2012 +0300 @@ -4,7 +4,7 @@ #include "ioloop.h" #include "array.h" #include "aqueue.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "hex-binary.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/db-ldap.c --- a/src/auth/db-ldap.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/db-ldap.c Wed Oct 03 18:17:26 2012 +0300 @@ -4,7 +4,7 @@ #if defined(BUILTIN_LDAP) || defined(PLUGIN_BUILD) -#include "network.h" +#include "net.h" #include "ioloop.h" #include "array.h" #include "hash.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/main.c --- a/src/auth/main.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/main.c Wed Oct 03 18:17:26 2012 +0300 @@ -3,7 +3,7 @@ #include "auth-common.h" #include "array.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "lib-signals.h" #include "restrict-access.h" #include "child-wait.h" diff -r 559718e321a2 -r 02451e967a06 src/auth/passdb-pam.c --- a/src/auth/passdb-pam.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/auth/passdb-pam.c Wed Oct 03 18:17:26 2012 +0300 @@ -15,7 +15,7 @@ #include "lib-signals.h" #include "str.h" #include "var-expand.h" -#include "network.h" +#include "net.h" #include "safe-memset.h" #include "auth-cache.h" diff -r 559718e321a2 -r 02451e967a06 src/config/config-filter.h --- a/src/config/config-filter.h Wed Oct 03 16:57:28 2012 +0300 +++ b/src/config/config-filter.h Wed Oct 03 18:17:26 2012 +0300 @@ -1,7 +1,7 @@ #ifndef CONFIG_FILTER_H #define CONFIG_FILTER_H -#include "network.h" +#include "net.h" struct master_service_settings_output; diff -r 559718e321a2 -r 02451e967a06 src/config/settings-get.pl --- a/src/config/settings-get.pl Wed Oct 03 16:57:28 2012 +0300 +++ b/src/config/settings-get.pl Wed Oct 03 18:17:26 2012 +0300 @@ -8,7 +8,7 @@ print '#include "file-lock.h"'."\n"; print '#include "fsync-mode.h"'."\n"; print '#include "hash-format.h"'."\n"; -print '#include "network.h"'."\n"; +print '#include "net.h"'."\n"; print '#include "unichar.h"'."\n"; print '#include "settings-parser.h"'."\n"; print '#include "all-settings.h"'."\n"; diff -r 559718e321a2 -r 02451e967a06 src/director/auth-connection.c --- a/src/director/auth-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/auth-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -4,7 +4,7 @@ #include "ioloop.h" #include "istream.h" #include "ostream.h" -#include "network.h" +#include "net.h" #include "llist.h" #include "safe-memset.h" #include "auth-client-interface.h" diff -r 559718e321a2 -r 02451e967a06 src/director/director-connection.c --- a/src/director/director-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/director-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -31,7 +31,7 @@ #include "lib.h" #include "ioloop.h" #include "array.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "str.h" diff -r 559718e321a2 -r 02451e967a06 src/director/director-host.h --- a/src/director/director-host.h Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/director-host.h Wed Oct 03 18:17:26 2012 +0300 @@ -1,7 +1,7 @@ #ifndef DIRECTOR_HOST_H #define DIRECTOR_HOST_H -#include "network.h" +#include "net.h" struct director; diff -r 559718e321a2 -r 02451e967a06 src/director/director.h --- a/src/director/director.h Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/director.h Wed Oct 03 18:17:26 2012 +0300 @@ -1,7 +1,7 @@ #ifndef DIRECTOR_H #define DIRECTOR_H -#include "network.h" +#include "net.h" #include "director-settings.h" #define DIRECTOR_VERSION_NAME "director" diff -r 559718e321a2 -r 02451e967a06 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/doveadm-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "array.h" diff -r 559718e321a2 -r 02451e967a06 src/director/login-connection.c --- a/src/director/login-connection.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/login-connection.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "ioloop.h" -#include "network.h" +#include "net.h" #include "ostream.h" #include "llist.h" #include "master-service.h" diff -r 559718e321a2 -r 02451e967a06 src/director/mail-host.h --- a/src/director/mail-host.h Wed Oct 03 16:57:28 2012 +0300 +++ b/src/director/mail-host.h Wed Oct 03 18:17:26 2012 +0300 @@ -1,7 +1,7 @@ #ifndef MAIL_HOST_H #define MAIL_HOST_H -#include "network.h" +#include "net.h" struct mail_host_list; diff -r 559718e321a2 -r 02451e967a06 src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/doveadm/doveadm-director.c Wed Oct 03 18:17:26 2012 +0300 @@ -4,7 +4,7 @@ #include "md5.h" #include "hash.h" #include "str.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "write-full.h" #include "master-service.h" diff -r 559718e321a2 -r 02451e967a06 src/doveadm/doveadm-kick.c --- a/src/doveadm/doveadm-kick.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/doveadm/doveadm-kick.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "array.h" -#include "network.h" +#include "net.h" #include "hash.h" #include "doveadm.h" #include "doveadm-who.h" diff -r 559718e321a2 -r 02451e967a06 src/doveadm/doveadm-mail-index.c --- a/src/doveadm/doveadm-mail-index.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/doveadm/doveadm-mail-index.c Wed Oct 03 18:17:26 2012 +0300 @@ -3,7 +3,7 @@ #include "lib.h" #include "str.h" #include "strescape.h" -#include "network.h" +#include "net.h" #include "write-full.h" #include "mail-namespace.h" #include "mail-storage.h" diff -r 559718e321a2 -r 02451e967a06 src/doveadm/doveadm-penalty.c --- a/src/doveadm/doveadm-penalty.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/doveadm/doveadm-penalty.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "array.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "hash.h" #include "doveadm.h" diff -r 559718e321a2 -r 02451e967a06 src/doveadm/doveadm-stats.c --- a/src/doveadm/doveadm-stats.c Wed Oct 03 16:57:28 2012 +0300 +++ b/src/doveadm/doveadm-stats.c Wed Oct 03 18:17:26 2012 +0300 @@ -2,7 +2,7 @@ #include "lib.h" #include "array.h" -#include "network.h" +#include "net.h" #include "ioloop.h" #include "istream.h" #include "hash.h" From pigeonhole at rename-it.nl Wed Oct 3 19:00:56 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 03 Oct 2012 18:00:56 +0200 Subject: dovecot-2.2-pigeonhole: Adjusted to changes in Dovecot network API. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/df0ac1d9be50 changeset: 1687:df0ac1d9be50 user: Stephan Bosch date: Wed Oct 03 18:00:50 2012 +0200 description: Adjusted to changes in Dovecot network API. diffstat: src/managesieve-login/client.h | 2 +- src/managesieve/managesieve-client.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 7d171ec86a4c -r df0ac1d9be50 src/managesieve-login/client.h --- a/src/managesieve-login/client.h Sun Sep 23 17:14:58 2012 +0200 +++ b/src/managesieve-login/client.h Wed Oct 03 18:00:50 2012 +0200 @@ -4,7 +4,7 @@ #ifndef __CLIENT_H #define __CLIENT_H -#include "network.h" +#include "net.h" #include "client-common.h" /* maximum length for managesieve command line. */ diff -r 7d171ec86a4c -r df0ac1d9be50 src/managesieve/managesieve-client.c --- a/src/managesieve/managesieve-client.c Sun Sep 23 17:14:58 2012 +0200 +++ b/src/managesieve/managesieve-client.c Wed Oct 03 18:00:50 2012 +0200 @@ -6,7 +6,7 @@ #include "llist.h" #include "str.h" #include "hostpid.h" -#include "network.h" +#include "net.h" #include "istream.h" #include "ostream.h" #include "var-expand.h" From dovecot at dovecot.org Thu Oct 4 00:56:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 04 Oct 2012 00:56:28 +0300 Subject: dovecot-2.2: Added/updated some copyright comments. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d66421812d2a changeset: 15188:d66421812d2a user: Timo Sirainen date: Thu Oct 04 00:56:14 2012 +0300 description: Added/updated some copyright comments. diffstat: AUTHORS | 2 ++ src/doveadm/doveadm-pw.c | 2 +- src/lib-imap/test-imap-url.c | 2 +- src/lib-sql/driver-sqlite.c | 2 +- src/lib/ioloop-epoll.c | 8 +------- src/lib/iso8601-date.c | 2 ++ src/lib/primes.c | 2 ++ src/lib/test-iso8601-date.c | 2 +- src/master/capabilities-posix.c | 2 ++ src/plugins/notify/notify-plugin.c | 2 ++ src/plugins/notify/notify-storage.c | 2 ++ 11 files changed, 17 insertions(+), 11 deletions(-) diffs (108 lines): diff -r 02451e967a06 -r d66421812d2a AUTHORS --- a/AUTHORS Wed Oct 03 18:17:26 2012 +0300 +++ b/AUTHORS Thu Oct 04 00:56:14 2012 +0300 @@ -8,6 +8,8 @@ Joshua Goodall (src/auth/mech-cram-md5.c, src/doveadm/doveadm-pw.c) +Jakob Hirsch (src/lib-sql/driver-sqlite.c) + Jelmer Vernooij (src/auth/mech-gssapi.c) Vaclav Haisman (src/lib/ioloop-kqueue.c, diff -r 02451e967a06 -r d66421812d2a src/doveadm/doveadm-pw.c --- a/src/doveadm/doveadm-pw.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/doveadm/doveadm-pw.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,4 +1,4 @@ -/* Copyright (C) 2004 Joshua Goodall */ +/* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "array.h" diff -r 02451e967a06 -r d66421812d2a src/lib-imap/test-imap-url.c --- a/src/lib-imap/test-imap-url.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib-imap/test-imap-url.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2011 Dovecot authors, see the included COPYING file */ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "net.h" diff -r 02451e967a06 -r d66421812d2a src/lib-sql/driver-sqlite.c --- a/src/lib-sql/driver-sqlite.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib-sql/driver-sqlite.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,4 +1,4 @@ -/* Copyright (C) 2006 Jakob Hirsch */ +/* Copyright (c) 2006-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "array.h" diff -r 02451e967a06 -r d66421812d2a src/lib/ioloop-epoll.c --- a/src/lib/ioloop-epoll.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib/ioloop-epoll.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,10 +1,4 @@ -/* - * Linux epoll() based ioloop handler. - * - * Copyright (c) 2004 Andrey Panin - * - * This software is released under the MIT license. - */ +/* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "array.h" diff -r 02451e967a06 -r d66421812d2a src/lib/iso8601-date.c --- a/src/lib/iso8601-date.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib/iso8601-date.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,3 +1,5 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + #include "lib.h" #include "utc-offset.h" #include "utc-mktime.h" diff -r 02451e967a06 -r d66421812d2a src/lib/primes.c --- a/src/lib/primes.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib/primes.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,3 +1,5 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + #include "lib.h" #include "primes.h" diff -r 02451e967a06 -r d66421812d2a src/lib/test-iso8601-date.c --- a/src/lib/test-iso8601-date.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/lib/test-iso8601-date.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2011 Dovecot authors, see the included COPYING file */ +/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ #include "test-lib.h" #include "test-common.h" diff -r 02451e967a06 -r d66421812d2a src/master/capabilities-posix.c --- a/src/master/capabilities-posix.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/master/capabilities-posix.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,3 +1,5 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + #include "common.h" #include "capabilities.h" diff -r 02451e967a06 -r d66421812d2a src/plugins/notify/notify-plugin.c --- a/src/plugins/notify/notify-plugin.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/plugins/notify/notify-plugin.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,3 +1,5 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + #include "lib.h" #include "llist.h" #include "mail-storage.h" diff -r 02451e967a06 -r d66421812d2a src/plugins/notify/notify-storage.c --- a/src/plugins/notify/notify-storage.c Wed Oct 03 18:17:26 2012 +0300 +++ b/src/plugins/notify/notify-storage.c Thu Oct 04 00:56:14 2012 +0300 @@ -1,3 +1,5 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + #include "lib.h" #include "array.h" #include "mail-storage-private.h" From dovecot at dovecot.org Thu Oct 4 02:35:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 04 Oct 2012 02:35:01 +0300 Subject: dovecot-2.2: dbox: Index rebuilding didn't open dovecot.index.ba... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa5c1d162714 changeset: 15189:aa5c1d162714 user: Timo Sirainen date: Thu Oct 04 02:08:23 2012 +0300 description: dbox: Index rebuilding didn't open dovecot.index.backup file. Recent changes broke it, and it was attemting to use dovecot.index/.backup file instead. diffstat: src/lib-storage/index/index-rebuild.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d66421812d2a -r aa5c1d162714 src/lib-storage/index/index-rebuild.c --- a/src/lib-storage/index/index-rebuild.c Thu Oct 04 00:56:14 2012 +0300 +++ b/src/lib-storage/index/index-rebuild.c Thu Oct 04 02:08:23 2012 +0300 @@ -170,7 +170,7 @@ if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &index_dir) <= 0) i_unreached(); - backup_path = t_strconcat(box->index_prefix, "/.backup", NULL); + backup_path = t_strconcat(box->index_prefix, ".backup", NULL); ctx->backup_index = mail_index_alloc(index_dir, backup_path); #ifndef MMAP_CONFLICTS_WRITE From dovecot at dovecot.org Thu Oct 4 02:35:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 04 Oct 2012 02:35:01 +0300 Subject: dovecot-2.2: lib-index: Simplified writing to dovecot.index.cach... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60c4815778fb changeset: 15190:60c4815778fb user: Timo Sirainen date: Thu Oct 04 02:34:53 2012 +0300 description: lib-index: Simplified writing to dovecot.index.cache file. The old method was basically: - write max. 32 kB to internal buffer - flush it by writing to reserved areas (with no locks) The reserved areas were acquired by doing (whenever needed): - lock dovecot.index.cache - reserve data from dovecot.index.cache for writing, potentially increasing the file size by writing 0 bytes. the reserved area size varies. - unlock dovecot.index.cache This worked, but if multiple processes were writing to the cache file it could have left incomplete reserved areas as holes. The holes were attempted to be filled if they were large enough. The new method is: - write max. 256 kB to internal buffer - lock dovecot.index.cache - append the buffer to dovecot.index.cache - unlock dovecot.index.cache No reserved areas, holes or anything else weird going on. Ideally no data would be overwritten in the dovecot.index.cache file, only appended. Unfortunately currently some data is still overwritten: - mail_cache_header.{deleted_space,continued_record_count} - mail_cache_header_fields.next_offset when writing a new one - mail_cache_header_fields.{last_used,decision} - mail_cache_record.prev_offset The changing headers could eventually be moved to dovecot.index. This however is a backwards-incompatible change. The record's prev_offset could maybe simply just not be written in those (somewhat rare) problematic situations. diffstat: src/doveadm/doveadm-dump-index.c | 4 +- src/lib-index/mail-cache-compress.c | 8 +- src/lib-index/mail-cache-private.h | 29 +- src/lib-index/mail-cache-transaction.c | 611 +++++--------------------------- src/lib-index/mail-cache.c | 68 ++- 5 files changed, 151 insertions(+), 569 deletions(-) diffs (truncated from 1033 to 300 lines): diff -r aa5c1d162714 -r 60c4815778fb src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Thu Oct 04 02:08:23 2012 +0300 +++ b/src/doveadm/doveadm-dump-index.c Thu Oct 04 02:34:53 2012 +0300 @@ -324,8 +324,8 @@ hdr->file_seq, unixdate2str(hdr->file_seq), hdr->file_seq - hdr->indexid); printf("continued_record_count = %u\n", hdr->continued_record_count); - printf("hole_offset .......... = %u\n", hdr->hole_offset); - printf("used_file_size ....... = %u\n", hdr->used_file_size); + printf("hole_offset (unused) . = %u\n", hdr->unused_old_hole_offset); + printf("used_file_size (old) . = %u\n", hdr->backwards_compat_used_file_size); printf("deleted_space ........ = %u\n", hdr->deleted_space); printf("field_header_offset .. = %u (0x%08x nontranslated)\n", mail_index_offset_to_uint32(hdr->field_header_offset), diff -r aa5c1d162714 -r 60c4815778fb src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Thu Oct 04 02:08:23 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 04 02:34:53 2012 +0300 @@ -272,7 +272,7 @@ mail_cache_compress_get_fields(&ctx, used_fields_count); o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used); - hdr.used_file_size = output->offset; + hdr.backwards_compat_used_file_size = output->offset; buffer_free(&ctx.buffer); buffer_free(&ctx.field_seen); @@ -287,12 +287,6 @@ array_free(ext_offsets); return -1; } - - if (hdr.used_file_size < MAIL_CACHE_INITIAL_SIZE) { - /* grow the file some more. doesn't matter if it fails */ - (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE); - } - o_stream_destroy(&output); if (cache->index->fsync_mode == FSYNC_MODE_ALWAYS) { diff -r aa5c1d162714 -r 60c4815778fb src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Thu Oct 04 02:08:23 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 04 02:34:53 2012 +0300 @@ -13,9 +13,6 @@ /* Never compress the file if it's smaller than this */ #define MAIL_CACHE_COMPRESS_MIN_SIZE (1024*50) -/* Don't bother remembering holes smaller than this */ -#define MAIL_CACHE_MIN_HOLE_SIZE 1024 - /* Compress the file when deleted space reaches n% of total size */ #define MAIL_CACHE_COMPRESS_PERCENTAGE 20 @@ -27,15 +24,6 @@ the latest cache header. */ #define MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT 4 -/* Initial size for the file */ -#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240) - -/* When more space is needed, grow the file n% larger than the previous size */ -#define MAIL_CACHE_GROW_PERCENTAGE 10 - -/* When allocating space for transactions, don't use blocks larger than this. */ -#define MAIL_CACHE_MAX_RESERVED_BLOCK_SIZE (1024*512) - #define MAIL_CACHE_LOCK_TIMEOUT 10 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 @@ -58,8 +46,8 @@ uint32_t continued_record_count; - uint32_t hole_offset; - uint32_t used_file_size; + uint32_t unused_old_hole_offset; + uint32_t backwards_compat_used_file_size; uint32_t deleted_space; uint32_t field_header_offset; @@ -102,17 +90,6 @@ /* array of { uint32_t field; [ uint32_t size; ] { .. } } */ }; -struct mail_cache_hole_header { - uint32_t next_offset; /* 0 if no holes left */ - uint32_t size; /* including this header */ - - /* make sure we notice if we're treating hole as mail_cache_record. - magic is a large number so if it's treated as size field, it'll - point outside the file */ -#define MAIL_CACHE_HOLE_HEADER_MAGIC 0xffeedeff - uint32_t magic; -}; - struct mail_cache_field_private { struct mail_cache_field field; @@ -230,6 +207,8 @@ int mail_cache_write(struct mail_cache *cache, const void *data, size_t size, uoff_t offset); +int mail_cache_append(struct mail_cache *cache, const void *data, size_t size, + uint32_t *offset_r); int mail_cache_header_fields_read(struct mail_cache *cache); int mail_cache_header_fields_update(struct mail_cache *cache); diff -r aa5c1d162714 -r 60c4815778fb src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Thu Oct 04 02:08:23 2012 +0300 +++ b/src/lib-index/mail-cache-transaction.c Thu Oct 04 02:34:53 2012 +0300 @@ -15,16 +15,12 @@ #include #include -#define MAIL_CACHE_WRITE_BUFFER 32768 +#define MAIL_CACHE_INIT_WRITE_BUFFER (1024*16) +#define MAIL_CACHE_MAX_WRITE_BUFFER (1024*256) #define CACHE_TRANS_CONTEXT(obj) \ MODULE_CONTEXT(obj, cache_mail_index_transaction_module) -struct mail_cache_reservation { - uint32_t offset; - uint32_t size; -}; - struct mail_cache_transaction_ctx { union mail_index_transaction_module_context module_ctx; struct mail_index_transaction_vfuncs super; @@ -39,11 +35,9 @@ buffer_t *cache_data; ARRAY(uint32_t) cache_data_seq; uint32_t prev_seq; - size_t prev_pos; + size_t last_rec_pos; - ARRAY(struct mail_cache_reservation) reservations; - uint32_t reserved_space_offset, reserved_space; - uint32_t last_grow_size; + uoff_t bytes_written; unsigned int tried_compression:1; unsigned int changes:1; @@ -52,10 +46,9 @@ static MODULE_CONTEXT_DEFINE_INIT(cache_mail_index_transaction_module, &mail_index_module_register); -static void -mail_cache_transaction_free_reservations(struct mail_cache_transaction_ctx *ctx); -static int mail_cache_link_unlocked(struct mail_cache *cache, - uint32_t old_offset, uint32_t new_offset); +static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx); +static int mail_cache_link_locked(struct mail_cache *cache, + uint32_t old_offset, uint32_t new_offset); static void mail_index_transaction_cache_reset(struct mail_index_transaction *t) { @@ -105,7 +98,6 @@ ctx->cache = view->cache; ctx->view = view; ctx->trans = t; - i_array_init(&ctx->reservations, 32); i_assert(view->transaction == NULL); view->transaction = ctx; @@ -132,23 +124,27 @@ if (array_is_created(&ctx->cache_data_seq)) array_clear(&ctx->cache_data_seq); ctx->prev_seq = 0; - ctx->prev_pos = 0; - - array_clear(&ctx->reservations); - ctx->reserved_space_offset = 0; - ctx->reserved_space = 0; - ctx->last_grow_size = 0; + ctx->last_rec_pos = 0; ctx->changes = FALSE; } -static void -mail_cache_transaction_free(struct mail_cache_transaction_ctx **_ctx) +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx) { struct mail_cache_transaction_ctx *ctx = *_ctx; *_ctx = NULL; + if (ctx->bytes_written > 0) { + /* we already wrote to the cache file. we can't (or don't want + to) delete that data, so just mark it as deleted space */ + if (mail_cache_transaction_lock(ctx) > 0) { + ctx->cache->hdr_copy.deleted_space += + ctx->bytes_written; + (void)mail_cache_unlock(ctx->cache); + } + } + MODULE_CONTEXT_UNSET(ctx->trans, cache_mail_index_transaction_module); ctx->view->transaction = NULL; @@ -159,7 +155,6 @@ buffer_free(&ctx->cache_data); if (array_is_created(&ctx->cache_data_seq)) array_free(&ctx->cache_data_seq); - array_free(&ctx->reservations); i_free(ctx); } @@ -279,346 +274,14 @@ return 1; } -static int mail_cache_grow_file(struct mail_cache *cache, size_t size) -{ - struct stat st; - uoff_t new_fsize, grow_size; - - i_assert(cache->locked); - - /* grow the file */ - new_fsize = cache->hdr_copy.used_file_size + size; - grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE; - if (grow_size < 16384) - grow_size = 16384; - new_fsize += grow_size; - new_fsize &= ~1023; - - if (fstat(cache->fd, &st) < 0) { - mail_cache_set_syscall_error(cache, "fstat()"); - return -1; - } - - if ((uoff_t)st.st_size < new_fsize) { - if (file_set_size(cache->fd, new_fsize) < 0) { - mail_cache_set_syscall_error(cache, "file_set_size()"); - return -1; - } - } - return 0; -} - -static bool mail_cache_unlink_hole(struct mail_cache *cache, size_t size, - struct mail_cache_hole_header *hole_r) -{ - struct mail_cache_header *hdr = &cache->hdr_copy; - struct mail_cache_hole_header hole; - uint32_t offset, prev_offset; - - i_assert(cache->locked); - - offset = hdr->hole_offset; prev_offset = 0; - while (offset != 0) { - if (pread_full(cache->fd, &hole, sizeof(hole), offset) <= 0) { - mail_cache_set_syscall_error(cache, "pread_full()"); - return FALSE; - } - - if (hole.magic != MAIL_CACHE_HOLE_HEADER_MAGIC) { - mail_cache_set_corrupted(cache, - "Invalid magic in hole header"); - return FALSE; - } - - if (hole.size >= size) - break; - - prev_offset = offset; - offset = hole.next_offset; - } - if (offset == 0) - return FALSE; - - if (prev_offset == 0) - hdr->hole_offset = hole.next_offset; - else { - if (mail_cache_write(cache, &hole.next_offset, - sizeof(hole.next_offset), prev_offset) < 0) - return FALSE; - } - hdr->deleted_space -= hole.size; - cache->hdr_modified = TRUE; - - hole_r->next_offset = offset; - hole_r->size = hole.size; - return TRUE; -} - -static void -mail_cache_transaction_add_reservation(struct mail_cache_transaction_ctx *ctx, - uint32_t offset, uint32_t size) -{ - struct mail_cache_reservation res; - - ctx->reserved_space_offset = offset; - ctx->reserved_space = size; - - res.offset = offset; - res.size = size; - - array_append(&ctx->reservations, &res, 1); From dovecot at dovecot.org Fri Oct 5 00:15:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 05 Oct 2012 00:15:12 +0300 Subject: dovecot-2.1: lib-storage: Don't crash when searching multiple ke... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0306792cc843 changeset: 14755:0306792cc843 user: Timo Sirainen date: Fri Oct 05 00:15:01 2012 +0300 description: lib-storage: Don't crash when searching multiple keywords. Fixed by simply removing the keyword merging code. mail_search_args_simplify() is called before mail_search_args_init(), so the keywords are still NULL and merging can't be done. Alternative fix would have been to add string array to mail_search_arg.value containing the keywords, but all of this is a pretty unnecessary optimization. diffstat: src/lib-storage/mail-search.c | 63 ------------------------------------------- 1 files changed, 0 insertions(+), 63 deletions(-) diffs (88 lines): diff -r 28be06dbd65e -r 0306792cc843 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Wed Oct 03 05:43:27 2012 +0300 +++ b/src/lib-storage/mail-search.c Fri Oct 05 00:15:01 2012 +0300 @@ -586,48 +586,14 @@ return TRUE; } -static struct mail_keywords * -mail_search_keywords_merge(struct mailbox *box, - struct mail_keywords **_kw1, - struct mail_keywords **_kw2) -{ - struct mail_keywords *kw1 = *_kw1, *kw2 = *_kw2; - struct mail_keywords *new_kw; - - i_assert(kw1->index == kw2->index); - T_BEGIN { - ARRAY_TYPE(keyword_indexes) new_indexes; - unsigned int i, j; - - t_array_init(&new_indexes, kw1->count + kw2->count + 1); - array_append(&new_indexes, kw1->idx, kw1->count); - for (i = 0; i < kw2->count; i++) { - /* don't add duplicates */ - for (j = 0; j < kw1->count; j++) { - if (kw1->idx[j] == kw2->idx[i]) - break; - } - if (j == kw1->count) - array_append(&new_indexes, kw2->idx+i, 1); - } - new_kw = mailbox_keywords_create_from_indexes(box, - &new_indexes); - } T_END; - mailbox_keywords_unref(_kw1); - mailbox_keywords_unref(_kw2); - return new_kw; -} - static void mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { struct mail_search_arg *sub, *prev = NULL; struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - struct mail_search_arg *prev_kw_arg, *prev_not_kw_arg; prev_flags_arg = prev_not_flags_arg = NULL; - prev_kw_arg = prev_not_kw_arg = NULL; while (args != NULL) { if (args->match_not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -689,35 +655,6 @@ } } - /* merge all keywords arguments */ - if (args->type == SEARCH_KEYWORDS && - !args->match_not && parent_and) { - if (prev_kw_arg == NULL) - prev_kw_arg = args; - else { - prev_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } else if (args->type == SEARCH_KEYWORDS && - args->match_not && !parent_and) { - if (prev_not_kw_arg == NULL) - prev_not_kw_arg = args; - else { - prev_not_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_not_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } - prev = args; args = args->next; } From dovecot at dovecot.org Fri Oct 5 00:25:14 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 05 Oct 2012 00:25:14 +0300 Subject: dovecot-2.0: lib-storage: Don't crash when searching multiple ke... Message-ID: details: http://hg.dovecot.org/dovecot-2.0/rev/13d764bbde11 changeset: 13107:13d764bbde11 user: Timo Sirainen date: Fri Oct 05 00:25:02 2012 +0300 description: lib-storage: Don't crash when searching multiple keywords. Fixed by simply removing the keyword merging code. mail_search_args_simplify() is called before mail_search_args_init(), so the keywords are still NULL and merging can't be done. Alternative fix would have been to add string array to mail_search_arg.value containing the keywords, but all of this is a pretty unnecessary optimization. diffstat: src/lib-storage/mail-search.c | 62 ------------------------------------------- 1 files changed, 0 insertions(+), 62 deletions(-) diffs (87 lines): diff -r 4c27b2418882 -r 13d764bbde11 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Tue Sep 04 20:27:10 2012 +0300 +++ b/src/lib-storage/mail-search.c Fri Oct 05 00:25:02 2012 +0300 @@ -592,48 +592,14 @@ return TRUE; } -static struct mail_keywords * -mail_search_keywords_merge(struct mailbox *box, - struct mail_keywords **_kw1, - struct mail_keywords **_kw2) -{ - struct mail_keywords *kw1 = *_kw1, *kw2 = *_kw2; - struct mail_keywords *new_kw; - - i_assert(kw1->index == kw2->index); - T_BEGIN { - ARRAY_TYPE(keyword_indexes) new_indexes; - unsigned int i, j; - - t_array_init(&new_indexes, kw1->count + kw2->count + 1); - array_append(&new_indexes, kw1->idx, kw1->count); - for (i = 0; i < kw2->count; i++) { - /* don't add duplicates */ - for (j = 0; j < kw1->count; j++) { - if (kw1->idx[j] == kw2->idx[i]) - break; - } - if (j == kw1->count) - array_append(&new_indexes, kw2->idx+i, 1); - } - new_kw = mailbox_keywords_create_from_indexes(box, - &new_indexes); - } T_END; - mailbox_keywords_unref(box, _kw1); - mailbox_keywords_unref(box, _kw2); - return new_kw; -} - static void mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { struct mail_search_arg *sub, *prev = NULL; struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - struct mail_search_arg *prev_kw_arg, *prev_not_kw_arg; prev_flags_arg = prev_not_flags_arg = NULL; - prev_kw_arg = prev_not_kw_arg = NULL; while (args != NULL) { if (args->not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -694,34 +660,6 @@ } } - /* merge all keywords arguments */ - if (args->type == SEARCH_KEYWORDS && !args->not && parent_and) { - if (prev_kw_arg == NULL) - prev_kw_arg = args; - else { - prev_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } else if (args->type == SEARCH_KEYWORDS && args->not && - !parent_and) { - if (prev_not_kw_arg == NULL) - prev_not_kw_arg = args; - else { - prev_not_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_not_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } - prev = args; args = args->next; } From dovecot at dovecot.org Fri Oct 5 00:26:24 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 05 Oct 2012 00:26:24 +0300 Subject: dovecot-1.2: lib-storage: Don't crash when searching multiple ke... Message-ID: details: http://hg.dovecot.org/dovecot-1.2/rev/d6bd9acd97e7 changeset: 9657:d6bd9acd97e7 user: Timo Sirainen date: Fri Oct 05 00:26:19 2012 +0300 description: lib-storage: Don't crash when searching multiple keywords. Fixed by simply removing the keyword merging code. mail_search_args_simplify() is called before mail_search_args_init(), so the keywords are still NULL and merging can't be done. Alternative fix would have been to add string array to mail_search_arg.value containing the keywords, but all of this is a pretty unnecessary optimization. diffstat: src/lib-storage/mail-search.c | 61 ------------------------------------------- 1 files changed, 0 insertions(+), 61 deletions(-) diffs (85 lines): diff -r 6862d534e5b1 -r d6bd9acd97e7 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Tue Jul 17 16:20:20 2012 +0300 +++ b/src/lib-storage/mail-search.c Fri Oct 05 00:26:19 2012 +0300 @@ -510,46 +510,13 @@ return buffer_get_data(headers, NULL); } -static struct mail_keywords * -mail_search_keywords_merge(struct mail_keywords **_kw1, - struct mail_keywords **_kw2) -{ - struct mail_keywords *kw1 = *_kw1, *kw2 = *_kw2; - struct mail_keywords *new_kw; - - i_assert(kw1->index == kw2->index); - T_BEGIN { - ARRAY_TYPE(keyword_indexes) new_indexes; - unsigned int i, j; - - t_array_init(&new_indexes, kw1->count + kw2->count + 1); - array_append(&new_indexes, kw1->idx, kw1->count); - for (i = 0; i < kw2->count; i++) { - /* don't add duplicates */ - for (j = 0; j < kw1->count; j++) { - if (kw1->idx[j] == kw2->idx[i]) - break; - } - if (j == kw1->count) - array_append(&new_indexes, kw2->idx+i, 1); - } - new_kw = mail_index_keywords_create_from_indexes(kw1->index, - &new_indexes); - } T_END; - mail_index_keywords_free(_kw1); - mail_index_keywords_free(_kw2); - return new_kw; -} - static void mail_search_args_simplify_sub(struct mail_search_arg *args, bool parent_and) { struct mail_search_arg *sub, *prev = NULL; struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - struct mail_search_arg *prev_kw_arg, *prev_not_kw_arg; prev_flags_arg = prev_not_flags_arg = NULL; - prev_kw_arg = prev_not_kw_arg = NULL; while (args != NULL) { if (args->not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -608,34 +575,6 @@ } } - /* merge all keywords arguments */ - if (args->type == SEARCH_KEYWORDS && !args->not && parent_and) { - if (prev_kw_arg == NULL) - prev_kw_arg = args; - else { - prev_kw_arg->value.keywords = - mail_search_keywords_merge( - &prev_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } else if (args->type == SEARCH_KEYWORDS && args->not && - !parent_and) { - if (prev_not_kw_arg == NULL) - prev_not_kw_arg = args; - else { - prev_not_kw_arg->value.keywords = - mail_search_keywords_merge( - &prev_not_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } - prev = args; args = args->next; } From dovecot at dovecot.org Mon Oct 8 00:01:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Oct 2012 00:01:12 +0300 Subject: dovecot-2.2: unlink_directory(): Added UNLINK_DIRECTORY_FLAG_SKI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9cb9c3cdfdc changeset: 15191:b9cb9c3cdfdc user: Timo Sirainen date: Mon Oct 08 00:00:55 2012 +0300 description: unlink_directory(): Added UNLINK_DIRECTORY_FLAG_SKIP_DOTFILES flag. diffstat: src/lib-storage/list/mailbox-list-delete.c | 2 +- src/lib/unlink-directory.c | 25 ++++++++++++++----------- src/lib/unlink-directory.h | 9 ++++++++- 3 files changed, 23 insertions(+), 13 deletions(-) diffs (108 lines): diff -r 60c4815778fb -r b9cb9c3cdfdc src/lib-storage/list/mailbox-list-delete.c --- a/src/lib-storage/list/mailbox-list-delete.c Thu Oct 04 02:34:53 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-delete.c Mon Oct 08 00:00:55 2012 +0300 @@ -313,7 +313,7 @@ int mailbox_list_delete_trash(const char *path) { - if (unlink_directory(path, TRUE) < 0) { + if (unlink_directory(path, UNLINK_DIRECTORY_FLAG_RMDIR) < 0) { if (errno == ELOOP) { /* it's a symlink? try just deleting it */ if (unlink(path) == 0) diff -r 60c4815778fb -r b9cb9c3cdfdc src/lib/unlink-directory.c --- a/src/lib/unlink-directory.c Thu Oct 04 02:34:53 2012 +0300 +++ b/src/lib/unlink-directory.c Mon Oct 08 00:00:55 2012 +0300 @@ -40,7 +40,8 @@ #include #include -static int unlink_directory_r(const char *dir) +static int +unlink_directory_r(const char *dir, enum unlink_directory_flags flags) { DIR *dirp; struct dirent *d; @@ -97,11 +98,14 @@ errno = 0; while ((d = readdir(dirp)) != NULL) { - if (d->d_name[0] == '.' && - (d->d_name[1] == '\0' || - (d->d_name[1] == '.' && d->d_name[2] == '\0'))) { - /* skip . and .. */ - continue; + if (d->d_name[0] == '.') { + if ((d->d_name[1] == '\0' || + (d->d_name[1] == '.' && d->d_name[2] == '\0'))) { + /* skip . and .. */ + continue; + } + if ((flags & UNLINK_DIRECTORY_FLAG_SKIP_DOTFILES) != 0) + continue; } if (unlink(d->d_name) < 0 && errno != ENOENT) { @@ -112,7 +116,7 @@ break; errno = 0; } else if (S_ISDIR(st.st_mode)) { - if (unlink_directory_r(d->d_name) < 0) { + if (unlink_directory_r(d->d_name, flags) < 0) { if (errno != ENOENT) break; errno = 0; @@ -159,7 +163,7 @@ return 0; } -int unlink_directory(const char *dir, bool unlink_dir) +int unlink_directory(const char *dir, enum unlink_directory_flags flags) { int fd, ret, old_errno; @@ -167,7 +171,7 @@ if (fd == -1) return -1; - ret = unlink_directory_r(dir); + ret = unlink_directory_r(dir, flags); if (ret < 0 && errno == ENOENT) ret = 0; old_errno = errno; @@ -183,7 +187,7 @@ return -1; } - if (unlink_dir) { + if ((flags & UNLINK_DIRECTORY_FLAG_RMDIR) != 0) { if (rmdir(dir) < 0 && errno != ENOENT) { if (errno == EEXIST) { /* standardize errno */ @@ -192,6 +196,5 @@ return -1; } } - return 0; } diff -r 60c4815778fb -r b9cb9c3cdfdc src/lib/unlink-directory.h --- a/src/lib/unlink-directory.h Thu Oct 04 02:34:53 2012 +0300 +++ b/src/lib/unlink-directory.h Mon Oct 08 00:00:55 2012 +0300 @@ -1,8 +1,15 @@ #ifndef UNLINK_DIRECTORY_H #define UNLINK_DIRECTORY_H +enum unlink_directory_flags { + /* After unlinking all files, rmdir() the directory itself */ + UNLINK_DIRECTORY_FLAG_RMDIR = 0x01, + /* Don't unlink any files beginning with "." */ + UNLINK_DIRECTORY_FLAG_SKIP_DOTFILES = 0x02 +}; + /* Unlink directory and/or everything under it. Returns 0 if successful, -1 if error. */ -int unlink_directory(const char *dir, bool unlink_dir); +int unlink_directory(const char *dir, enum unlink_directory_flags flags); #endif From dovecot at dovecot.org Mon Oct 8 00:49:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Oct 2012 00:49:11 +0300 Subject: dovecot-2.2: fts-lucene: Compile fix for previous unlink_directo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e63a281eeff9 changeset: 15192:e63a281eeff9 user: Timo Sirainen date: Mon Oct 08 00:48:55 2012 +0300 description: fts-lucene: Compile fix for previous unlink_directory() change diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r b9cb9c3cdfdc -r e63a281eeff9 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Mon Oct 08 00:00:55 2012 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon Oct 08 00:48:55 2012 +0300 @@ -238,7 +238,9 @@ err.number() == CL_ERR_IO)) { /* delete corrupted index. most IO errors are also about missing files and other such corruption.. */ - if (unlink_directory(index->path, TRUE) < 0 && errno != ENOENT) + if (unlink_directory(index->path, + UNLINK_DIRECTORY_FLAG_RMDIR) < 0 && + errno != ENOENT) i_error("unlink_directory(%s) failed: %m", index->path); rescan_clear_unseen_mailboxes(index, NULL); } @@ -361,7 +363,7 @@ return ret; /* settings changed, rebuild index */ - if (unlink_directory(index->path, TRUE) < 0) { + if (unlink_directory(index->path, UNLINK_DIRECTORY_FLAG_RMDIR) < 0) { i_error("unlink_directory(%s) failed: %m", index->path); ret = -1; } else { From dovecot at dovecot.org Mon Oct 8 03:14:24 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Oct 2012 03:14:24 +0300 Subject: dovecot-2.1: fts-lucene: doveadm fts rescan crashed with mailbox... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/d96313048f17 changeset: 14756:d96313048f17 user: Timo Sirainen date: Mon Oct 08 03:14:12 2012 +0300 description: fts-lucene: doveadm fts rescan crashed with mailbox_list_index=yes diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 0306792cc843 -r d96313048f17 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Fri Oct 05 00:15:01 2012 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon Oct 08 03:14:12 2012 +0300 @@ -753,7 +753,8 @@ while ((info = mailbox_list_iter_next(iter)) != NULL) { box = mailbox_alloc(index->list, info->name, (enum mailbox_flags)0); - if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + if (mailbox_open(box) == 0 && + mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) == 0 && (guids == NULL || hash_table_lookup(guids, metadata.guid) == NULL)) { From dovecot at dovecot.org Mon Oct 8 08:54:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Oct 2012 08:54:11 +0300 Subject: dovecot-2.1: master: If service { protocol } is set and not incl... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/4d268e810c15 changeset: 14757:4d268e810c15 user: Timo Sirainen date: Mon Oct 08 08:53:54 2012 +0300 description: master: If service { protocol } is set and not included in "protocols", ignore its settings diffstat: src/master/master-settings.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (28 lines): diff -r d96313048f17 -r 4d268e810c15 src/master/master-settings.c --- a/src/master/master-settings.c Mon Oct 08 03:14:12 2012 +0300 +++ b/src/master/master-settings.c Mon Oct 08 08:53:54 2012 +0300 @@ -496,6 +496,13 @@ for (i = 0; i < count; i++) { struct service_settings *service = services[i]; + if (*service->protocol != '\0' && + !str_array_find((const char **)set->protocols_split, + service->protocol)) { + /* protocol not enabled, ignore its settings */ + continue; + } + if (*service->executable == '\0') { *error_r = t_strdup_printf("service(%s): " "executable is empty", service->name); @@ -550,9 +557,7 @@ } #endif - if (*service->protocol != '\0' && - str_array_find((const char **)set->protocols_split, - service->protocol)) { + if (*service->protocol != '\0') { /* each imap/pop3/lmtp process can use up a connection, although if service_count=1 it's only temporary */ if (service->service_count != 1 || From dovecot at dovecot.org Thu Oct 11 00:02:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Oct 2012 00:02:28 +0300 Subject: dovecot-2.2: uri-util: Fix handling of '..' and '.' segments in ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f960ad98429c changeset: 15193:f960ad98429c user: Stephan Bosch date: Wed Oct 10 23:55:21 2012 +0300 description: uri-util: Fix handling of '..' and '.' segments in URI paths. As specified by RFC 3986 diffstat: src/lib/uri-util.c | 44 +++++++++++++++++++++++++++++++++----------- 1 files changed, 33 insertions(+), 11 deletions(-) diffs (89 lines): diff -r e63a281eeff9 -r f960ad98429c src/lib/uri-util.c --- a/src/lib/uri-util.c Mon Oct 08 00:48:55 2012 +0300 +++ b/src/lib/uri-util.c Wed Oct 10 23:55:21 2012 +0300 @@ -594,19 +594,24 @@ int uri_parse_path(struct uri_parser *parser, int *relative_r, const char *const **path_r) { + const unsigned char *pbegin = parser->cur; ARRAY_TYPE(const_string) segments; - const char *segment; + const char *segment = NULL; unsigned int count; int relative = 1; int ret; t_array_init(&segments, 16); + /* check for a leading '/' and indicate absolute path + when it is present + */ if (parser->cur < parser->end && *parser->cur == '/') { parser->cur++; relative = 0; } + /* parse first segment */ if ((ret = uri_parse_path_segment(parser, &segment)) < 0) return -1; @@ -615,32 +620,49 @@ /* strip dot segments */ if (segment[0] == '.') { if (segment[1] == '.') { - /* '..' -> pop last segment (if any) */ - count = array_count(&segments); - if (count > 0) { - array_delete(&segments, count-1, 1); - } else if ( relative > 0 ) { - relative++; + if (segment[2] == '\0') { + /* '..' -> skip and... */ + segment = NULL; + + /* ... pop last segment (if any) */ + count = array_count(&segments); + if (count > 0) { + array_delete(&segments, count-1, 1); + } else if ( relative > 0 ) { + relative++; + } } - } else { + } else if (segment[1] == '\0') { /* '.' -> skip */ + segment = NULL; } - } else { - array_append(&segments, &segment, 1); } } else { segment = ""; + } + + if (segment != NULL) array_append(&segments, &segment, 1); - } if (parser->cur >= parser->end || *parser->cur != '/') break; parser->cur++; + /* parse next path segment */ if ((ret = uri_parse_path_segment(parser, &segment)) < 0) return -1; } + if (parser->cur == pbegin) { + /* path part of URI is missing */ + return 0; + } + + /* special treatment for a trailing '..' or '.' */ + if (segment == NULL) { + segment = ""; + array_append(&segments, &segment, 1); + } array_append_zero(&segments); *path_r = array_get(&segments, &count); *relative_r = relative; From dovecot at dovecot.org Thu Oct 11 00:02:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Oct 2012 00:02:28 +0300 Subject: dovecot-2.2: Adjust IMAP URL parser to changes in uri-util. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8929da80876 changeset: 15194:b8929da80876 user: Stephan Bosch date: Wed Oct 10 23:56:01 2012 +0300 description: Adjust IMAP URL parser to changes in uri-util. diffstat: src/lib-imap/imap-url.c | 10 ++++++---- src/lib-imap/test-imap-url.c | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diffs (62 lines): diff -r f960ad98429c -r b8929da80876 src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Wed Oct 10 23:55:21 2012 +0300 +++ b/src/lib-imap/imap-url.c Wed Oct 10 23:56:01 2012 +0300 @@ -726,10 +726,12 @@ } if (*segment != NULL) { - parser->error = t_strdup_printf( - "Unexpected IMAP URL path segment: %s", - str_sanitize(*segment, 80)); - return -1; + if (urlext != NULL || **segment != '\0' || *(segment+1) != NULL ) { + parser->error = t_strdup_printf( + "Unexpected IMAP URL path segment: `%s'", + str_sanitize(*segment, 80)); + return -1; + } } /* ";" {...} at end of URL */ diff -r f960ad98429c -r b8929da80876 src/lib-imap/test-imap-url.c --- a/src/lib-imap/test-imap-url.c Wed Oct 10 23:55:21 2012 +0300 +++ b/src/lib-imap/test-imap-url.c Wed Oct 10 23:56:01 2012 +0300 @@ -114,7 +114,7 @@ .url_parsed = { .host_name = "example.com", .userid = "user", - .mailbox = "INBOX" } + .mailbox = "INBOX/" } },{ .url = "imap://user at example.com/INBOX/Trash/../", .url_parsed = { @@ -176,7 +176,7 @@ .url_parsed = { .host_name = "example.com", .userid = "user", - .mailbox = "INBOX", .uidvalidity = 0, + .mailbox = "INBOX/", .uidvalidity = 0, .uid = 0 } },{ .url = "imap://user at example.com/INBOX/Junk;UIDVALIDITY=27667/" @@ -207,7 +207,7 @@ .host_name = "example.com", .userid = "user", .mailbox = "INBOX/Important", - .uid = 56, .section = "AA" } + .uid = 56, .section = "AA/" } },{ .url = "imap://user at example.com/INBOX/Important/;UID=56/" ";SECTION=AA/BB/../..", @@ -897,8 +897,10 @@ const char *parse_create_url_tests[] = { "imap://host.example.com/", + "imap://10.0.0.1/", +#ifdef HAVE_IPV6 "imap://[::1]/", - "imap://10.0.0.1/", +#endif "imap://user at host.example.com/", "imap://user at host.example.com:993/", "imap://user;AUTH=PLAIN at host.example.com/", From dovecot at dovecot.org Thu Oct 11 00:02:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Oct 2012 00:02:28 +0300 Subject: dovecot-2.2: Add HTTP date parsing support. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d927aaaf9252 changeset: 15196:d927aaaf9252 user: Stephan Bosch date: Wed Oct 10 23:59:12 2012 +0300 description: Add HTTP date parsing support. diffstat: src/lib-http/Makefile.am | 7 + src/lib-http/http-date.c | 497 +++++++++++++++++++++++++++++++++ src/lib-http/http-date.h | 17 + src/lib-http/test-http-date.c | 222 ++++++++++++++ src/lib-http/test-http-header-parser.c | 224 ++++++++++++++ 5 files changed, 967 insertions(+), 0 deletions(-) diffs (truncated from 1009 to 300 lines): diff -r 70305d850220 -r d927aaaf9252 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Wed Oct 10 23:57:56 2012 +0300 +++ b/src/lib-http/Makefile.am Wed Oct 10 23:59:12 2012 +0300 @@ -7,15 +7,18 @@ -I$(top_srcdir)/src/lib-ssl-iostream libhttp_la_SOURCES = \ + http-date.c \ http-url.c headers = \ + http-date.h \ http-url.h pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) test_programs = \ + test-http-date \ test-http-url noinst_PROGRAMS = $(test_programs) @@ -30,6 +33,10 @@ test_http_url_LDADD = http-url.lo $(test_libs) test_http_url_DEPENDENCIES = $(test_deps) +test_http_date_SOURCES = test-http-date.c +test_http_date_LDADD = http-date.lo $(test_libs) +test_http_date_DEPENDENCIES = $(test_deps) + check: check-am check-test check-test: all-am for bin in $(test_programs); do \ diff -r 70305d850220 -r d927aaaf9252 src/lib-http/http-date.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-date.c Wed Oct 10 23:59:12 2012 +0300 @@ -0,0 +1,497 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "utc-mktime.h" +#include "http-date.h" + +#include + +/* + Official specification is still RFC261, Section 3.3, but we anticipate + HTTPbis and use the draft Part 2, Section 5.1 as reference for our + parser: + + http://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-20#section-5.1 + + The defined syntax is as follows: + + HTTP-date = rfc1123-date / obs-date + + Preferred format: + + rfc1123-date = day-name "," SP date1 SP time-of-day SP GMT + ; fixed length subset of the format defined in + ; Section 5.2.14 of [RFC1123] + day-name = %x4D.6F.6E ; "Mon", case-sensitive + / %x54.75.65 ; "Tue", case-sensitive + / %x57.65.64 ; "Wed", case-sensitive + / %x54.68.75 ; "Thu", case-sensitive + / %x46.72.69 ; "Fri", case-sensitive + / %x53.61.74 ; "Sat", case-sensitive + / %x53.75.6E ; "Sun", case-sensitive + date1 = day SP month SP year + ; e.g., 02 Jun 1982 + day = 2DIGIT + month = %x4A.61.6E ; "Jan", case-sensitive + / %x46.65.62 ; "Feb", case-sensitive + / %x4D.61.72 ; "Mar", case-sensitive + / %x41.70.72 ; "Apr", case-sensitive + / %x4D.61.79 ; "May", case-sensitive + / %x4A.75.6E ; "Jun", case-sensitive + / %x4A.75.6C ; "Jul", case-sensitive + / %x41.75.67 ; "Aug", case-sensitive + / %x53.65.70 ; "Sep", case-sensitive + / %x4F.63.74 ; "Oct", case-sensitive + / %x4E.6F.76 ; "Nov", case-sensitive + / %x44.65.63 ; "Dec", case-sensitive + year = 4DIGIT + GMT = %x47.4D.54 ; "GMT", case-sensitive + time-of-day = hour ":" minute ":" second + ; 00:00:00 - 23:59:59 + hour = 2DIGIT + minute = 2DIGIT + second = 2DIGIT + + The semantics of day-name, day, month, year, and time-of-day are the + same as those defined for the RFC 5322 constructs with the + corresponding name ([RFC5322], Section 3.3). + + Obsolete formats: + + obs-date = rfc850-date / asctime-date + + rfc850-date = day-name-l "," SP date2 SP time-of-day SP GMT + date2 = day "-" month "-" 2DIGIT + ; day-month-year (e.g., 02-Jun-82) + day-name-l = %x4D.6F.6E.64.61.79 ; "Monday", case-sensitive + / %x54.75.65.73.64.61.79 ; "Tuesday", case-sensitive + / %x57.65.64.6E.65.73.64.61.79 ; "Wednesday", case-sensitive + / %x54.68.75.72.73.64.61.79 ; "Thursday", case-sensitive + / %x46.72.69.64.61.79 ; "Friday", case-sensitive + / %x53.61.74.75.72.64.61.79 ; "Saturday", case-sensitive + / %x53.75.6E.64.61.79 ; "Sunday", case-sensitive + + asctime-date = day-name SP date3 SP time-of-day SP year + date3 = month SP ( 2DIGIT / ( SP 1DIGIT )) + ; month day (e.g., Jun 2) + + */ + +static const char *month_names[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +static const char *weekday_names[] = { + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" +}; + +static const char *weekday_names_long[] = { + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" +}; + +struct http_date_parser { + const unsigned char *cur, *end; + + struct tm tm; + int timezone_offset; +}; + +static inline int +http_date_parse_sp(struct http_date_parser *parser) +{ + if (parser->cur >= parser->end) + return -1; + if (parser->cur[0] != ' ') + return 0; + parser->cur++; + return 1; +} + +static inline int +http_date_parse_number(struct http_date_parser *parser, + int digits, int *number_r) +{ + int i; + + if (parser->cur >= parser->end || !i_isdigit(parser->cur[0])) + return 0; + + *number_r = parser->cur[0] - '0'; + parser->cur++; + + for (i=0; i < digits-1; i++) { + if (parser->cur >= parser->end || !i_isdigit(parser->cur[0])) + return -1; + *number_r = ((*number_r) * 10) + parser->cur[0] - '0'; + parser->cur++; + } + return 1; +} + +static inline int +http_date_parse_word(struct http_date_parser *parser, + int maxchars, string_t **word_r) +{ + string_t *word; + int i; + + if (parser->cur >= parser->end || !i_isalpha(parser->cur[0])) + return 0; + + word = t_str_new(maxchars); + str_append_c(word, parser->cur[0]); + parser->cur++; + + for (i=0; i < maxchars-1; i++) { + if (parser->cur >= parser->end || !i_isalpha(parser->cur[0])) + break; + str_append_c(word, parser->cur[0]); + parser->cur++; + } + + if (i_isalpha(parser->cur[0])) + return -1; + *word_r = word; + return 1; +} + +static inline int +http_date_parse_year(struct http_date_parser *parser) +{ + /* year = 4DIGIT */ + if (http_date_parse_number(parser, 4, &parser->tm.tm_year) <= 0) + return -1; + if (parser->tm.tm_year < 1900) + return -1; + parser->tm.tm_year -= 1900; + return 1; +} + +static inline int +http_date_parse_month(struct http_date_parser *parser) +{ + string_t *month; + int i; + + if (http_date_parse_word(parser, 3, &month) <= 0 || str_len(month) != 3) + return -1; + + for (i = 0; i < 12; i++) { + if (strcmp(month_names[i], str_c(month)) == 0) { + break; + } + } + if (i >= 12) + return -1; + + parser->tm.tm_mon = i; + return 1; +} + +static inline int +http_date_parse_day(struct http_date_parser *parser) +{ + /* day = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_mday) <= 0) + return -1; + return 1; +} + +static int +http_date_parse_time_of_day(struct http_date_parser *parser) +{ + /* time-of-day = hour ":" minute ":" second + ; 00:00:00 - 23:59:59 + hour = 2DIGIT + minute = 2DIGIT + second = 2DIGIT + */ + + /* hour = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_hour) <= 0) + return -1; + + /* ":" */ + if (parser->cur >= parser->end || parser->cur[0] != ':') + return -1; + parser->cur++; + + /* minute = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_min) <= 0) + return -1; + + /* ":" */ + if (parser->cur >= parser->end || parser->cur[0] != ':') + return -1; + parser->cur++; + + /* second = 2DIGIT */ + if (http_date_parse_number(parser, 2, &parser->tm.tm_sec) <= 0) + return -1; + return 1; +} + +static inline int +http_date_parse_time_gmt(struct http_date_parser *parser) +{ + string_t *gmt; + + /* Remaining: {...} SP time-of-day SP GMT + */ + + /* SP time-of-day */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_time_of_day(parser) <= 0) + return -1; + + /* SP GMT */ + if (http_date_parse_sp(parser) <= 0) + return -1; + if (http_date_parse_word(parser, 3, &gmt) <= 0 || + strcmp("GMT", str_c(gmt)) != 0) + return -1; + return 1; +} + +static int +http_date_parse_format_rfc1123(struct http_date_parser *parser) +{ + /* + rfc1123-date = day-name "," SP date1 SP time-of-day SP GMT From dovecot at dovecot.org Thu Oct 11 00:02:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Oct 2012 00:02:28 +0300 Subject: dovecot-2.2: Adds HTTP URL parse support. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/70305d850220 changeset: 15195:70305d850220 user: Stephan Bosch date: Wed Oct 10 23:57:56 2012 +0300 description: Adds HTTP URL parse support. diffstat: configure.in | 1 + src/Makefile.am | 1 + src/lib-http/Makefile.am | 37 +++ src/lib-http/http-url.c | 287 ++++++++++++++++++++++++++ src/lib-http/http-url.h | 46 ++++ src/lib-http/test-http-url.c | 462 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 834 insertions(+), 0 deletions(-) diffs (truncated from 870 to 300 lines): diff -r b8929da80876 -r 70305d850220 configure.in --- a/configure.in Wed Oct 10 23:56:01 2012 +0300 +++ b/configure.in Wed Oct 10 23:57:56 2012 +0300 @@ -2754,6 +2754,7 @@ src/lib-dict/Makefile src/lib-dns/Makefile src/lib-fs/Makefile +src/lib-http/Makefile src/lib-imap/Makefile src/lib-imap-storage/Makefile src/lib-imap-client/Makefile diff -r b8929da80876 -r 70305d850220 src/Makefile.am --- a/src/Makefile.am Wed Oct 10 23:56:01 2012 +0300 +++ b/src/Makefile.am Wed Oct 10 23:57:56 2012 +0300 @@ -9,6 +9,7 @@ lib-imap \ lib-imap-storage \ lib-master \ + lib-http \ lib-dict \ lib-settings \ lib-ssl-iostream diff -r b8929da80876 -r 70305d850220 src/lib-http/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/Makefile.am Wed Oct 10 23:57:56 2012 +0300 @@ -0,0 +1,37 @@ +noinst_LTLIBRARIES = libhttp.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-ssl-iostream + +libhttp_la_SOURCES = \ + http-url.c + +headers = \ + http-url.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +test_programs = \ + test-http-url + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_deps = $(noinst_LTLIBRARIES) $(test_libs) + +test_http_url_SOURCES = test-http-url.c +test_http_url_LDADD = http-url.lo $(test_libs) +test_http_url_DEPENDENCIES = $(test_deps) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff -r b8929da80876 -r 70305d850220 src/lib-http/http-url.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-url.c Wed Oct 10 23:57:56 2012 +0300 @@ -0,0 +1,287 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strfuncs.h" +#include "net.h" +#include "uri-util.h" +#include "http-url.h" + +/* + * HTTP URL parser + */ + +struct http_url_parser { + struct uri_parser parser; + + enum http_url_parse_flags flags; + + struct http_url *url; + struct http_url *base; + + unsigned int relative:1; +}; + +static bool http_url_do_parse(struct http_url_parser *url_parser) +{ + struct uri_parser *parser = &url_parser->parser; + struct http_url *url = url_parser->url, *base = url_parser->base; + struct uri_authority auth; + const char *const *path; + bool relative = TRUE, have_path = FALSE; + int path_relative; + const char *part; + int ret; + + /* RFC 2616 - Hypertext Transfer Protocol, Section 3.2: + * + * http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + * + * Translated to RFC 3986: + * + * absolute-http-URL = "http:" "//" host [ ":" port ] path-absolute + * ["?" query] [ "#" fragment ] + * relative-http-ref = relative-http-part [ "?" query ] [ "#" fragment ] + * relative-http-part = "//" host [ ":" port ] path-abempty + * / path-absolute + * / path-noscheme + * / path-empty + */ + + /* "http:" / "https:" */ + if ((url_parser->flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0) { + const char *scheme; + + if ((ret = uri_parse_scheme(parser, &scheme)) < 0) + return FALSE; + else if (ret > 0) { + if (strcasecmp(scheme, "https") == 0) { + if (url != NULL) + url->have_ssl = TRUE; + } else if (strcasecmp(scheme, "http") != 0) { + parser->error = "Not an HTTP URL"; + return FALSE; + } + relative = FALSE; + } + } else { + relative = FALSE; + } + + /* "//" host [ ":" port ] */ + if ((ret = uri_parse_authority(parser, &auth)) < 0) + return FALSE; + if (ret > 0) { + if (auth.enc_userinfo != NULL) { + /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20 + + Section 2.8.1: + + {...} Senders MUST NOT include a userinfo subcomponent (and its "@" + delimiter) when transmitting an "http" URI in a message. Recipients + of HTTP messages that contain a URI reference SHOULD parse for the + existence of userinfo and treat its presence as an error, likely + indicating that the deprecated subcomponent is being used to + obscure the authority for the sake of phishing attacks. + */ + parser->error = "HTTP URL does not allow `userinfo@' part"; + return FALSE; + } + relative = FALSE; + } else if (!relative) { + parser->error = "Absolute HTTP URL requires `//' after `http:'"; + return FALSE; + } + + if (ret > 0 && url != NULL) { + url->host_name = auth.host_literal; + url->host_ip = auth.host_ip; + url->have_host_ip = auth.have_host_ip; + url->port = auth.port; + url->have_port = auth.have_port; + } + + /* path-abempty / path-absolute / path-noscheme / path-empty */ + if ((ret = uri_parse_path(parser, &path_relative, &path)) < 0) + return FALSE; + + /* Relative URLs are only valid when we have a base URL */ + if (relative) { + if (base == NULL) { + parser->error = "Relative URL not allowed"; + return FALSE; + } else if (url != NULL) { + url->host_name = p_strdup_empty(parser->pool, base->host_name); + url->host_ip = base->host_ip; + url->have_host_ip = base->have_host_ip; + url->port = base->port; + url->have_port = base->have_port; + } + + url_parser->relative = TRUE; + } + + /* Resolve path */ + if (ret > 0) { + string_t *fullpath; + + have_path = TRUE; + + if (url != NULL) + fullpath = t_str_new(256); + + if (relative && path_relative > 0 && base->path != NULL) { + const char *pbegin = base->path; + const char *pend = base->path + strlen(base->path); + const char *p = pend - 1; + + i_assert(*pbegin == '/'); + + /* discard trailing segments of base path based on how many effective + leading '..' segments were found in the relative path. + */ + while (path_relative > 0 && p > pbegin) { + while (p > pbegin && *p != '/') p--; + if (p >= pbegin) { + pend = p; + path_relative--; + } + if (p > pbegin) p--; + } + + if (url != NULL && pend > pbegin) + str_append_n(fullpath, pbegin, pend-pbegin); + } + + /* append relative path */ + while (*path != NULL) { + if (!uri_data_decode(parser, *path, NULL, &part)) + return FALSE; + + if (url != NULL) { + str_append_c(fullpath, '/'); + str_append(fullpath, part); + } + path++; + } + + if (url != NULL) + url->path = str_c(fullpath); + } else if (relative && url != NULL) { + url->path = base->path; + } + + /* [ "?" query ] */ + if ((ret = uri_parse_query(parser, &part)) < 0) + return FALSE; + if (ret > 0) { + if (!uri_data_decode(parser, part, NULL, NULL)) // check only + return FALSE; + if (url != NULL) + url->enc_query = p_strdup(parser->pool, part); + } else if (relative && !have_path && url != NULL) { + url->enc_query = p_strdup(parser->pool, base->enc_query); + } + + /* [ "#" fragment ] */ + if ((ret = uri_parse_fragment(parser, &part)) < 0) + return FALSE; + if (ret > 0) { + if ((url_parser->flags & HTTP_URL_ALLOW_FRAGMENT_PART) == 0) { + parser->error = "URL fragment not allowed for HTTP URL in this context"; + return FALSE; + } + if (!uri_data_decode(parser, part, NULL, NULL)) // check only + return FALSE; + if (url != NULL) + url->enc_fragment = p_strdup(parser->pool, part); + } else if (relative && !have_path && url != NULL) { + url->enc_fragment = p_strdup(parser->pool, base->enc_fragment); + } + + if (parser->cur != parser->end) { + parser->error = "HTTP URL contains invalid character."; + return FALSE; + } + return TRUE; +} + +/* Public API */ + +int http_url_parse(const char *url, struct http_url *base, + enum http_url_parse_flags flags, + struct http_url **url_r, const char **error_r) +{ + struct http_url_parser url_parser; + + /* base != NULL indicates whether relative URLs are allowed. However, certain + flags may also dictate whether relative URLs are allowed/required. */ + i_assert((flags & HTTP_URL_PARSE_SCHEME_EXTERNAL) == 0 || base == NULL); + + memset(&url_parser, '\0', sizeof(url_parser)); + uri_parser_init(&url_parser.parser, pool_datastack_create(), url); + + url_parser.url = t_new(struct http_url, 1); + url_parser.base = base; + url_parser.flags = flags; + + if (!http_url_do_parse(&url_parser)) { + *error_r = url_parser.parser.error; + return -1; + } + *url_r = url_parser.url; + return 0; From pigeonhole at rename-it.nl Thu Oct 11 02:48:39 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 11 Oct 2012 01:48:39 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: fileinto/keep: don't log quot... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/7301a7ed8800 changeset: 1656:7301a7ed8800 user: Stephan Bosch date: Thu Oct 11 01:48:29 2012 +0200 description: lib-sieve: fileinto/keep: don't log quota errors to syslog but rather to user's log. Prevents administrator frustration about useless log messages caused by users with too much mail. diffstat: src/lib-sieve/sieve-actions.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 1963a329c539 -r 7301a7ed8800 src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Wed Sep 26 21:52:32 2012 +0200 +++ b/src/lib-sieve/sieve-actions.c Thu Oct 11 01:48:29 2012 +0200 @@ -604,8 +604,9 @@ (mailbox_get_storage(trans->box), &error_code); } - if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS ) - { + if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS && + error_code != MAIL_ERROR_NOSPACE) + { sieve_result_global_error(aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); } else { From pigeonhole at rename-it.nl Thu Oct 11 21:26:09 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 11 Oct 2012 20:26:09 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: Further improved handling of ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/5c1ce25596ed changeset: 1657:5c1ce25596ed user: Stephan Bosch date: Thu Oct 11 20:26:03 2012 +0200 description: lib-sieve: Further improved handling of quota errors. Added means to log user errors/warnings as info in master log. Previous change was inadequate because an error was still logged. diffstat: src/lib-sieve/sieve-actions.c | 14 +++-- src/lib-sieve/sieve-error-private.h | 3 +- src/lib-sieve/sieve-error.c | 76 ++++++++++++++++++++++++++++--- src/lib-sieve/sieve-error.h | 17 +++++++ src/lib-sieve/sieve-result.c | 21 ++++++++- src/lib-sieve/sieve-result.h | 6 ++ src/plugins/lda-sieve/lda-sieve-plugin.c | 18 ++++++- 7 files changed, 136 insertions(+), 19 deletions(-) diffs (truncated from 303 to 300 lines): diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-actions.c Thu Oct 11 20:26:03 2012 +0200 @@ -604,14 +604,16 @@ (mailbox_get_storage(trans->box), &error_code); } - if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS && - error_code != MAIL_ERROR_NOSPACE) - { + if ( error_code == MAIL_ERROR_NOTFOUND || + error_code == MAIL_ERROR_PARAMS ) { + sieve_result_error(aenv, "failed to store into mailbox %s: %s", + mailbox_name, errstr); + } else if ( error_code == MAIL_ERROR_NOSPACE ) { + sieve_result_global_log_error + (aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); + } else { sieve_result_global_error(aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); - } else { - sieve_result_error(aenv, "failed to store into mailbox %s: %s", - mailbox_name, errstr); } /* Store aborted? */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error-private.h --- a/src/lib-sieve/sieve-error-private.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error-private.h Thu Oct 11 20:26:03 2012 +0200 @@ -11,7 +11,8 @@ */ enum sieve_error_flags { - SIEVE_ERROR_FLAG_GLOBAL = (1 << 0) + SIEVE_ERROR_FLAG_GLOBAL = (1 << 0), + SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO = (1 << 1), }; /* diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error.c Thu Oct 11 20:26:03 2012 +0200 @@ -84,11 +84,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->verror - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->verror + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || sieve_errors_more_allowed(ehandler) ) { if ( ehandler->verror != NULL ) @@ -111,11 +117,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->vwarning - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->vwarning + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->vwarning != NULL ) ehandler->vwarning(ehandler, flags, location, fmt, args); @@ -140,7 +152,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_info ) { if ( ehandler->vinfo != NULL ) @@ -164,7 +177,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_debug ) { if ( ehandler->vdebug != NULL ) @@ -300,6 +314,24 @@ (svinst, ehandler, SIEVE_ERROR_FLAG_GLOBAL, location, fmt, args); } +void sieve_global_info_verror +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_verror(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + +void sieve_global_info_vwarning +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_vwarning(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + void sieve_global_error (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) @@ -342,6 +374,34 @@ va_end(args); } +void sieve_global_info_error +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + + T_BEGIN { + sieve_global_info_verror(svinst, ehandler, location, fmt, args); + } T_END; + + va_end(args); +} + +void sieve_global_info_warning +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + + T_BEGIN { + sieve_global_info_vwarning(svinst, ehandler, location, fmt, args); + } T_END; + + va_end(args); +} + /* * Default (user) error functions */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error.h --- a/src/lib-sieve/sieve-error.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error.h Thu Oct 11 20:26:03 2012 +0200 @@ -30,6 +30,11 @@ (struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) ATTR_FORMAT(3, 4); +typedef void (*sieve_sys_error_vfunc_t) + (struct sieve_instance *svinst, const char *fmt, va_list args); +typedef void (*sieve_sys_error_func_t) + (struct sieve_instance *svinst, const char *fmt, ...) ATTR_FORMAT(2, 3); + /* * System errors */ @@ -70,6 +75,12 @@ void sieve_global_vinfo (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, va_list args); +void sieve_global_info_verror + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args); +void sieve_global_info_vwarning + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args); void sieve_global_error (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, @@ -80,6 +91,12 @@ void sieve_global_info (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); +void sieve_global_info_error + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); +void sieve_global_info_warning + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); /* * Main (user) error functions diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-result.c --- a/src/lib-sieve/sieve-result.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-result.c Thu Oct 11 20:26:03 2012 +0200 @@ -272,7 +272,6 @@ va_end(args); } - void sieve_result_log (const struct sieve_action_exec_env *aenv, const char *fmt, ...) { @@ -293,6 +292,26 @@ va_end(args); } +void sieve_result_global_log_error +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + sieve_global_info_verror(aenv->svinst, aenv->ehandler, NULL, fmt, args); + va_end(args); +} + +void sieve_result_global_log_warning +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + sieve_global_info_vwarning(aenv->svinst, aenv->ehandler, NULL, fmt, args); + va_end(args); +} + /* * Result composition */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-result.h --- a/src/lib-sieve/sieve-result.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-result.h Thu Oct 11 20:26:03 2012 +0200 @@ -105,6 +105,12 @@ void sieve_result_global_log (const struct sieve_action_exec_env *aenv, const char *fmt, ...) ATTR_FORMAT(2, 3); +void sieve_result_global_log_error +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) + ATTR_FORMAT(2, 3); +void sieve_result_global_log_warning +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) + ATTR_FORMAT(2, 3); /* * Result composition diff -r 7301a7ed8800 -r 5c1ce25596ed src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Thu Oct 11 20:26:03 2012 +0200 @@ -353,17 +353,29 @@ (struct lda_sieve_run_context *srctx, struct sieve_script *script, int status) { struct sieve_instance *svinst = srctx->svinst; + struct sieve_exec_status *estatus = srctx->scriptenv->exec_status; const char *userlog_notice = ""; + sieve_sys_error_func_t error_func = sieve_sys_error; int ret; + if ( estatus != NULL && estatus->last_storage != NULL ) { + enum mail_error mail_error; + + mail_storage_get_last_error(estatus->last_storage, &mail_error); + + /* Don't bother administrator too much with benign errors */ + if ( mail_error == MAIL_ERROR_NOSPACE ) + error_func = sieve_sys_info; + } + if ( script == srctx->user_script && srctx->userlog != NULL ) { userlog_notice = t_strdup_printf - (" (user logfile %s may reveal additional details)", srctx->userlog); + (" (user logfile %s should reveal additional details)", srctx->userlog); } switch ( status ) { case SIEVE_EXEC_FAILURE: - sieve_sys_error(svinst, + error_func(svinst, "execution of script %s failed, but implicit keep was successful%s", sieve_script_location(script), userlog_notice); ret = 1; @@ -376,7 +388,7 @@ ret = -1; break; case SIEVE_EXEC_KEEP_FAILED: - sieve_sys_error(svinst, + error_func(svinst, From dovecot at dovecot.org Thu Oct 11 23:01:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Oct 2012 23:01:27 +0300 Subject: dovecot-2.2: Compiler warning fix. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2ac184a82b9a changeset: 15197:2ac184a82b9a user: Timo Sirainen date: Thu Oct 11 23:01:13 2012 +0300 description: Compiler warning fix. diffstat: src/lib-http/http-url.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d927aaaf9252 -r 2ac184a82b9a src/lib-http/http-url.c --- a/src/lib-http/http-url.c Wed Oct 10 23:59:12 2012 +0300 +++ b/src/lib-http/http-url.c Thu Oct 11 23:01:13 2012 +0300 @@ -123,7 +123,7 @@ /* Resolve path */ if (ret > 0) { - string_t *fullpath; + string_t *fullpath = NULL; have_path = TRUE; From pigeonhole at rename-it.nl Thu Oct 11 23:07:59 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 11 Oct 2012 22:07:59 +0200 Subject: dovecot-2.1-pigeonhole: ManageSieve: fixed handling of unkown co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/ceef02768dee changeset: 1658:ceef02768dee user: Stephan Bosch date: Thu Oct 11 22:07:50 2012 +0200 description: ManageSieve: fixed handling of unkown commands pre-login. Forgot to skip line upon error. diffstat: src/managesieve-login/client.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 5c1ce25596ed -r ceef02768dee src/managesieve-login/client.c --- a/src/managesieve-login/client.c Thu Oct 11 20:26:03 2012 +0200 +++ b/src/managesieve-login/client.c Thu Oct 11 22:07:50 2012 +0200 @@ -195,6 +195,8 @@ if ( cmd->name != NULL ) client->cmd = cmd; + else + client->skip_line = TRUE; } if ( client->cmd != NULL && !client->cmd_parsed_args ) { From dovecot at dovecot.org Fri Oct 12 00:22:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 00:22:36 +0300 Subject: dovecot-2.2: lib-fs API cleanups and improvements Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce57bacc3010 changeset: 15198:ce57bacc3010 user: Timo Sirainen date: Fri Oct 12 00:22:19 2012 +0300 description: lib-fs API cleanups and improvements diffstat: src/lib-fs/fs-api-private.h | 47 ++- src/lib-fs/fs-api.c | 127 +++++-- src/lib-fs/fs-api.h | 122 +++++- src/lib-fs/fs-posix.c | 373 ++++++++++++++++------ src/lib-fs/fs-sis-common.c | 25 +- src/lib-fs/fs-sis-common.h | 3 +- src/lib-fs/fs-sis-queue.c | 226 ++++++++----- src/lib-fs/fs-sis.c | 304 +++++++++++------- src/lib-storage/index/dbox-common/dbox-storage.c | 1 + src/lib-storage/index/dbox-single/sdbox-copy.c | 9 +- src/lib-storage/index/dbox-single/sdbox-file.c | 19 +- src/lib-storage/index/index-attachment.c | 43 +- 12 files changed, 855 insertions(+), 444 deletions(-) diffs (truncated from 2319 to 300 lines): diff -r 2ac184a82b9a -r ce57bacc3010 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu Oct 11 23:01:13 2012 +0300 +++ b/src/lib-fs/fs-api-private.h Fri Oct 12 00:22:19 2012 +0300 @@ -4,14 +4,29 @@ #include "fs-api.h" struct fs_vfuncs { - int (*init)(const char *args, const struct fs_settings *set, - struct fs **fs_r, const char **error_r); + struct fs *(*alloc)(void); + int (*init)(struct fs *fs, const char *args, + const struct fs_settings *set); void (*deinit)(struct fs *fs); - int (*open)(struct fs *fs, const char *path, enum fs_open_mode mode, - enum fs_open_flags flags, struct fs_file **file_r); - void (*close)(struct fs_file *file); + enum fs_properties (*get_properties)(struct fs *fs); + struct fs_file *(*file_init)(struct fs *fs, const char *path, + enum fs_open_mode mode, + enum fs_open_flags flags); + void (*file_deinit)(struct fs_file *file); + + void (*set_async_callback)(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context); + void (*wait_async)(struct fs *fs); + + void (*set_metadata)(struct fs_file *file, const char *key, + const char *value); + int (*get_metadata)(struct fs_file *file, + const ARRAY_TYPE(fs_metadata) **metadata_r); + + bool (*prefetch)(struct fs_file *file, uoff_t length); ssize_t (*read)(struct fs_file *file, void *buf, size_t size); struct istream *(*read_stream)(struct fs_file *file, size_t max_buffer_size); @@ -23,14 +38,16 @@ int (*lock)(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r); void (*unlock)(struct fs_lock *lock); - int (*fdatasync)(struct fs_file *file); - int (*exists)(struct fs *fs, const char *path); - int (*stat)(struct fs *fs, const char *path, struct stat *st_r); - int (*link)(struct fs *fs, const char *src, const char *dest); - int (*rename)(struct fs *fs, const char *src, const char *dest); - int (*unlink)(struct fs *fs, const char *path); - int (*rmdir)(struct fs *fs, const char *path); + int (*exists)(struct fs_file *file); + int (*stat)(struct fs_file *file, struct stat *st_r); + int (*copy)(struct fs_file *src, struct fs_file *dest); + int (*rename)(struct fs_file *src, struct fs_file *dest); + int (*delete_file)(struct fs_file *file); + + struct fs_iter *(*iter_init)(struct fs *fs, const char *path); + const char *(*iter_next)(struct fs_iter *iter); + int (*iter_deinit)(struct fs_iter *iter); }; struct fs { @@ -53,6 +70,10 @@ struct fs_file *file; }; +struct fs_iter { + struct fs *fs; +}; + extern struct fs fs_class_posix; extern struct fs fs_class_sis; extern struct fs fs_class_sis_queue; @@ -60,4 +81,6 @@ void fs_set_error(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); void fs_set_critical(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); +void fs_set_error_async(struct fs *fs); + #endif diff -r 2ac184a82b9a -r ce57bacc3010 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Oct 11 23:01:13 2012 +0300 +++ b/src/lib-fs/fs-api.c Fri Oct 12 00:22:19 2012 +0300 @@ -14,26 +14,24 @@ fs_alloc(const struct fs *fs_class, const char *args, const struct fs_settings *set, struct fs **fs_r, const char **error_r) { - struct fs *fs = NULL; - char *error_dup = NULL; + struct fs *fs; int ret; + fs = fs_class->v.alloc(); + fs->last_error = str_new(default_pool, 64); + T_BEGIN { - const char *error; - - ret = fs_class->v.init(args, set, &fs, &error); - if (ret < 0) - error_dup = i_strdup(error); + ret = fs_class->v.init(fs, args, set); } T_END; if (ret < 0) { /* a bit kludgy way to allow data stack frame usage in normal conditions but still be able to return error message from data stack. */ - *error_r = t_strdup_printf("%s: %s", fs_class->name, error_dup); - i_free(error_dup); + *error_r = t_strdup_printf("%s: %s", fs_class->name, + fs_last_error(fs)); + fs_deinit(&fs); return -1; } - fs->last_error = str_new(default_pool, 64); *fs_r = fs; return 0; } @@ -57,6 +55,7 @@ void fs_deinit(struct fs **_fs) { struct fs *fs = *_fs; + string_t *last_error = fs->last_error; *_fs = NULL; @@ -65,25 +64,25 @@ fs->name, fs->files_open_count); } - str_free(&fs->last_error); fs->v.deinit(fs); + str_free(&last_error); } -int fs_open(struct fs *fs, const char *path, int mode_flags, - struct fs_file **file_r) +struct fs_file *fs_file_init(struct fs *fs, const char *path, int mode_flags) { - int ret; + struct fs_file *file; + + i_assert(path != NULL); T_BEGIN { - ret = fs->v.open(fs, path, mode_flags & FS_OPEN_MODE_MASK, - mode_flags & ~FS_OPEN_MODE_MASK, file_r); + file = fs->v.file_init(fs, path, mode_flags & FS_OPEN_MODE_MASK, + mode_flags & ~FS_OPEN_MODE_MASK); } T_END; - if (ret == 0) - fs->files_open_count++; - return ret; + fs->files_open_count++; + return file; } -void fs_close(struct fs_file **_file) +void fs_file_deinit(struct fs_file **_file) { struct fs_file *file = *_file; @@ -92,7 +91,28 @@ *_file = NULL; file->fs->files_open_count--; - file->fs->v.close(file); + file->fs->v.file_deinit(file); +} + +enum fs_properties fs_get_properties(struct fs *fs) +{ + return fs->v.get_properties(fs); +} + +void fs_set_metadata(struct fs_file *file, const char *key, const char *value) +{ + if (file->fs->v.set_metadata != NULL) + file->fs->v.set_metadata(file, key, value); +} + +int fs_get_metadata(struct fs_file *file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + if (file->fs->v.get_metadata == NULL) { + fs_set_error(file->fs, "Metadata not supported by backend"); + return -1; + } + return file->fs->v.get_metadata(file, metadata_r); } const char *fs_file_path(struct fs_file *file) @@ -112,6 +132,11 @@ return fs_last_error(file->fs); } +bool fs_prefetch(struct fs_file *file, uoff_t length) +{ + return file->fs->v.prefetch(file, length); +} + ssize_t fs_read(struct fs_file *file, void *buf, size_t size) { return file->fs->v.read(file, buf, size); @@ -150,6 +175,22 @@ (void)file->fs->v.write_stream_finish(file, FALSE); } +void fs_file_set_async_callback(struct fs_file *file, + fs_file_async_callback_t *callback, + void *context) +{ + if (file->fs->v.set_async_callback != NULL) + file->fs->v.set_async_callback(file, callback, context); + else + callback(context); +} + +void fs_wait_async(struct fs *fs) +{ + if (fs->v.wait_async != NULL) + fs->v.wait_async(fs); +} + int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r) { return file->fs->v.lock(file, secs, lock_r); @@ -163,39 +204,49 @@ lock->file->fs->v.unlock(lock); } -int fs_fdatasync(struct fs_file *file) +int fs_exists(struct fs_file *file) { - return file->fs->v.fdatasync(file); + return file->fs->v.exists(file); } -int fs_exists(struct fs *fs, const char *path) +int fs_stat(struct fs_file *file, struct stat *st_r) { - return fs->v.exists(fs, path); + return file->fs->v.stat(file, st_r); } -int fs_stat(struct fs *fs, const char *path, struct stat *st_r) +int fs_copy(struct fs_file *src, struct fs_file *dest) { - return fs->v.stat(fs, path, st_r); + i_assert(src->fs == dest->fs); + return src->fs->v.copy(src, dest); } -int fs_link(struct fs *fs, const char *src, const char *dest) +int fs_rename(struct fs_file *src, struct fs_file *dest) { - return fs->v.link(fs, src, dest); + i_assert(src->fs == dest->fs); + return src->fs->v.rename(src, dest); } -int fs_rename(struct fs *fs, const char *src, const char *dest) +int fs_delete(struct fs_file *file) { - return fs->v.rename(fs, src, dest); + return file->fs->v.delete_file(file); } -int fs_unlink(struct fs *fs, const char *path) +struct fs_iter *fs_iter_init(struct fs *fs, const char *path) { - return fs->v.unlink(fs, path); + return fs->v.iter_init(fs, path); } -int fs_rmdir(struct fs *fs, const char *path) +int fs_iter_deinit(struct fs_iter **_iter) { - return fs->v.rmdir(fs, path); + struct fs_iter *iter = *_iter; + + *_iter = NULL; + return iter->fs->v.iter_deinit(iter); +} + +const char *fs_iter_next(struct fs_iter *iter) +{ + return iter->fs->v.iter_next(iter); } void fs_set_error(struct fs *fs, const char *fmt, ...) @@ -218,3 +269,9 @@ i_error("fs-%s: %s", fs->name, str_c(fs->last_error)); va_end(args); } + +void fs_set_error_async(struct fs *fs) +{ + fs_set_error(fs, "Asynchronous operation in progress"); + errno = EAGAIN; From dovecot at dovecot.org Fri Oct 12 00:30:34 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 00:30:34 +0300 Subject: dovecot-2.1: sdbox: Class didn't have MAIL_STORAGE_CLASS_FLAG_FI... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7bc24fc1667e changeset: 14758:7bc24fc1667e user: Timo Sirainen date: Fri Oct 12 00:29:41 2012 +0300 description: sdbox: Class didn't have MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG set This currently only meant that mail_prefetch_count setting wasn't working. diffstat: src/lib-storage/index/dbox-single/sdbox-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4d268e810c15 -r 7bc24fc1667e src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 08 08:53:54 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Fri Oct 12 00:29:41 2012 +0300 @@ -369,7 +369,7 @@ struct mail_storage sdbox_storage = { .name = SDBOX_STORAGE_NAME, - .class_flags = 0, + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG, .v = { NULL, From dovecot at dovecot.org Fri Oct 12 00:30:34 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 00:30:34 +0300 Subject: dovecot-2.1: sdbox: Make sure mail_attachment_fs=sis-queue isn't... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/9542732069ff changeset: 14759:9542732069ff user: Timo Sirainen date: Fri Oct 12 00:30:23 2012 +0300 description: sdbox: Make sure mail_attachment_fs=sis-queue isn't attempted to be used. It could be fixed, but nobody seems to have used it so far.. diffstat: src/lib-storage/index/dbox-common/dbox-storage.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (18 lines): diff -r 7bc24fc1667e -r 9542732069ff src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Fri Oct 12 00:29:41 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Fri Oct 12 00:30:23 2012 +0300 @@ -99,6 +99,14 @@ } else { name = t_strdup_until(set->mail_attachment_fs, args++); } + if (strcmp(name, "sis-queue") == 0 && + (_storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) != 0) { + /* FIXME: the deduplication part doesn't work, because + sdbox renames the files.. */ + *error_r = "mail_attachment_fs: " + "sis-queue not currently supported by sdbox"; + return -1; + } dir = mail_user_home_expand(_storage->user, set->mail_attachment_dir); storage->attachment_dir = p_strdup(_storage->pool, dir); From dovecot at dovecot.org Fri Oct 12 00:52:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 00:52:06 +0300 Subject: dovecot-2.2: Plugin ABI version checking improvements. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/73916b7be94e changeset: 15199:73916b7be94e user: Timo Sirainen date: Fri Oct 12 00:51:41 2012 +0300 description: Plugin ABI version checking improvements. Previously the plugin version was checked against the version string returned by the currently running Dovecot master process, not necessarily the same as the binary. Also version_ignore=yes setting skipped the version check entirely. Now there's a new DOVECOT_ABI_VERSION macro that can (at least in theory) be updated only when the ABI actually changes. The version is in format "2.2.ABIv1(2.2.15)", where the (2.2.15) would be the actual Dovecot version number that gets ignored when comparing the strings. Also now the plugin version is compared to the actually running binary's ABI, not the master's version, and it can't be ignored with a setting. diffstat: configure.in | 1 + src/auth/main.c | 4 ++-- src/config/config-parser.c | 2 +- src/dict/main.c | 2 +- src/doveadm/doveadm-mail.c | 2 +- src/doveadm/doveadm-util.c | 2 +- src/lib-sql/driver-mysql.c | 2 +- src/lib-sql/driver-pgsql.c | 2 +- src/lib-sql/driver-sqlite.c | 2 +- src/lib-storage/mail-storage-service.c | 2 +- src/lib/module-dir.c | 19 +++++++++++++++---- src/lib/module-dir.h | 8 +++++--- src/plugins/acl/acl-plugin.c | 2 +- src/plugins/acl/doveadm-acl.c | 2 +- src/plugins/expire/doveadm-expire.c | 2 +- src/plugins/expire/expire-plugin.c | 2 +- src/plugins/fts-lucene/doveadm-fts-lucene.c | 2 +- src/plugins/fts-lucene/fts-lucene-plugin.c | 2 +- src/plugins/fts-solr/fts-solr-plugin.c | 2 +- src/plugins/fts-squat/fts-squat-plugin.c | 2 +- src/plugins/fts/doveadm-fts.c | 2 +- src/plugins/fts/fts-plugin.c | 2 +- src/plugins/imap-acl/imap-acl-plugin.c | 2 +- src/plugins/imap-quota/imap-quota-plugin.c | 2 +- src/plugins/imap-stats/imap-stats-plugin.c | 2 +- src/plugins/imap-zlib/imap-zlib-plugin.c | 2 +- src/plugins/lazy-expunge/lazy-expunge-plugin.c | 2 +- src/plugins/listescape/listescape-plugin.c | 2 +- src/plugins/mailbox-alias/mailbox-alias-plugin.c | 2 +- src/plugins/notify/notify-plugin.c | 2 +- src/plugins/pop3-migration/pop3-migration-plugin.c | 2 +- src/plugins/quota/doveadm-quota.c | 2 +- src/plugins/quota/quota-plugin.c | 2 +- src/plugins/snarf/snarf-plugin.c | 2 +- src/plugins/stats/stats-plugin.c | 2 +- src/plugins/trash/trash-plugin.c | 2 +- src/plugins/virtual/virtual-plugin.c | 2 +- src/plugins/zlib/doveadm-zlib.c | 2 +- src/plugins/zlib/zlib-plugin.c | 2 +- 39 files changed, 58 insertions(+), 44 deletions(-) diffs (truncated from 507 to 300 lines): diff -r ce57bacc3010 -r 73916b7be94e configure.in --- a/configure.in Fri Oct 12 00:22:19 2012 +0300 +++ b/configure.in Fri Oct 12 00:51:41 2012 +0300 @@ -1,5 +1,6 @@ AC_PREREQ([2.59]) AC_INIT([Dovecot],[2.2.UNSTABLE],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv0($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) diff -r ce57bacc3010 -r 73916b7be94e src/auth/main.c --- a/src/auth/main.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/auth/main.c Fri Oct 12 00:51:41 2012 +0300 @@ -188,7 +188,7 @@ services = read_global_settings(); memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = global_auth_settings->debug; mod_set.filter_callback = auth_module_filter; @@ -218,7 +218,7 @@ struct module_dir_load_settings mod_set; memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = global_auth_settings->debug; mod_set.ignore_missing = TRUE; diff -r ce57bacc3010 -r 73916b7be94e src/config/config-parser.c --- a/src/config/config-parser.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/config/config-parser.c Fri Oct 12 00:51:41 2012 +0300 @@ -989,7 +989,7 @@ unsigned int i, count; memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; modules = module_dir_load(CONFIG_MODULE_DIR, NULL, &mod_set); module_dir_init(modules); diff -r ce57bacc3010 -r 73916b7be94e src/dict/main.c --- a/src/dict/main.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/dict/main.c Fri Oct 12 00:51:41 2012 +0300 @@ -55,7 +55,7 @@ } memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.require_init_funcs = TRUE; modules = module_dir_load(DICT_MODULE_DIR, NULL, &mod_set); diff -r ce57bacc3010 -r 73916b7be94e src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/doveadm/doveadm-mail.c Fri Oct 12 00:51:41 2012 +0300 @@ -703,7 +703,7 @@ doveadm_mail_register_cmd(mail_commands[i]); memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = doveadm_debug; mod_set.binary_name = "doveadm"; diff -r ce57bacc3010 -r 73916b7be94e src/doveadm/doveadm-util.c --- a/src/doveadm/doveadm-util.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/doveadm/doveadm-util.c Fri Oct 12 00:51:41 2012 +0300 @@ -25,7 +25,7 @@ only those whose dependencies have been loaded earlier, the rest are ignored. */ memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(master_service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = doveadm_debug; mod_set.ignore_dlopen_errors = TRUE; diff -r ce57bacc3010 -r 73916b7be94e src/lib-sql/driver-mysql.c --- a/src/lib-sql/driver-mysql.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib-sql/driver-mysql.c Fri Oct 12 00:51:41 2012 +0300 @@ -659,7 +659,7 @@ .failed_try_retry = TRUE }; -const char *driver_mysql_version = DOVECOT_VERSION; +const char *driver_mysql_version = DOVECOT_ABI_VERSION; void driver_mysql_init(void); void driver_mysql_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib-sql/driver-pgsql.c Fri Oct 12 00:51:41 2012 +0300 @@ -1087,7 +1087,7 @@ } }; -const char *driver_pgsql_version = DOVECOT_VERSION; +const char *driver_pgsql_version = DOVECOT_ABI_VERSION; void driver_pgsql_init(void); void driver_pgsql_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/lib-sql/driver-sqlite.c --- a/src/lib-sql/driver-sqlite.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib-sql/driver-sqlite.c Fri Oct 12 00:51:41 2012 +0300 @@ -444,7 +444,7 @@ } }; -const char *driver_sqlite_version = DOVECOT_VERSION; +const char *driver_sqlite_version = DOVECOT_ABI_VERSION; void driver_sqlite_init(void); void driver_sqlite_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib-storage/mail-storage-service.c Fri Oct 12 00:51:41 2012 +0300 @@ -918,7 +918,7 @@ return; memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = master_service_get_version_string(ctx->service); + mod_set.abi_version = DOVECOT_ABI_VERSION; mod_set.binary_name = master_service_get_name(ctx->service); mod_set.setting_name = "mail_plugins"; mod_set.require_init_funcs = TRUE; diff -r ce57bacc3010 -r 73916b7be94e src/lib/module-dir.c --- a/src/lib/module-dir.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib/module-dir.c Fri Oct 12 00:51:41 2012 +0300 @@ -168,6 +168,17 @@ #endif } +static bool versions_equal(const char *str1, const char *str2) +{ + while (*str1 == *str2) { + if (*str1 == '\0' || *str1 == '(') + return TRUE; + str1++; + str2++; + } + return FALSE; +} + static struct module * module_load(const char *path, const char *name, const struct module_dir_load_settings *set, @@ -212,12 +223,12 @@ module->name = i_strdup(name); module->handle = handle; - module_version = set->version == NULL ? NULL : + module_version = set->abi_version == NULL ? NULL : get_symbol(module, t_strconcat(name, "_version", NULL), TRUE); if (module_version != NULL && - strcmp(*module_version, set->version) != 0) { - i_error("Module is for different version %s: %s", - *module_version, path); + !versions_equal(*module_version, set->abi_version)) { + i_error("Module is for different ABI version %s (we have %s): %s", + *module_version, set->abi_version, path); module_free(module); return NULL; } diff -r ce57bacc3010 -r 73916b7be94e src/lib/module-dir.h --- a/src/lib/module-dir.h Fri Oct 12 00:22:19 2012 +0300 +++ b/src/lib/module-dir.h Fri Oct 12 00:51:41 2012 +0300 @@ -2,9 +2,11 @@ #define MODULE_DIR_H struct module_dir_load_settings { - /* If version is non-NULL and the module contains a version symbol, - fail the load if they're different. */ - const char *version; + /* If abi_version is non-NULL and the module contains a version symbol, + fail the load if they're different. In both strings ignore anything + after the first '(' character, so the version can be e.g.: + 2.2.ABIv1(2.2.15) */ + const char *abi_version; /* Binary name used for checking if plugin is tried to be loaded for wrong binary. */ const char *binary_name; diff -r ce57bacc3010 -r 73916b7be94e src/plugins/acl/acl-plugin.c --- a/src/plugins/acl/acl-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/acl/acl-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -7,7 +7,7 @@ #include -const char *acl_plugin_version = DOVECOT_VERSION; +const char *acl_plugin_version = DOVECOT_ABI_VERSION; static struct mail_storage_hooks acl_mail_storage_hooks = { .mail_user_created = acl_mail_user_created, diff -r ce57bacc3010 -r 73916b7be94e src/plugins/acl/doveadm-acl.c --- a/src/plugins/acl/doveadm-acl.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/acl/doveadm-acl.c Fri Oct 12 00:51:41 2012 +0300 @@ -16,7 +16,7 @@ enum acl_modify_mode modify_mode; }; -const char *doveadm_acl_plugin_version = DOVECOT_VERSION; +const char *doveadm_acl_plugin_version = DOVECOT_ABI_VERSION; void doveadm_acl_plugin_init(struct module *module); void doveadm_acl_plugin_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/expire/doveadm-expire.c --- a/src/plugins/expire/doveadm-expire.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/expire/doveadm-expire.c Fri Oct 12 00:51:41 2012 +0300 @@ -40,7 +40,7 @@ bool delete_nonexistent_users; }; -const char *doveadm_expire_plugin_version = DOVECOT_VERSION; +const char *doveadm_expire_plugin_version = DOVECOT_ABI_VERSION; void doveadm_expire_plugin_init(struct module *module); void doveadm_expire_plugin_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/expire/expire-plugin.c --- a/src/plugins/expire/expire-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/expire/expire-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -39,7 +39,7 @@ unsigned int first_expunged:1; }; -const char *expire_plugin_version = DOVECOT_VERSION; +const char *expire_plugin_version = DOVECOT_ABI_VERSION; static MODULE_CONTEXT_DEFINE_INIT(expire_storage_module, &mail_storage_module_register); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts-lucene/doveadm-fts-lucene.c --- a/src/plugins/fts-lucene/doveadm-fts-lucene.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts-lucene/doveadm-fts-lucene.c Fri Oct 12 00:51:41 2012 +0300 @@ -8,7 +8,7 @@ #include #include -const char *doveadm_fts_lucene_plugin_version = DOVECOT_VERSION; +const char *doveadm_fts_lucene_plugin_version = DOVECOT_ABI_VERSION; void doveadm_fts_lucene_plugin_init(struct module *module); void doveadm_fts_lucene_plugin_deinit(void); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -6,7 +6,7 @@ #include "lucene-wrapper.h" #include "fts-lucene-plugin.h" -const char *fts_lucene_plugin_version = DOVECOT_VERSION; +const char *fts_lucene_plugin_version = DOVECOT_ABI_VERSION; struct fts_lucene_user_module fts_lucene_user_module = MODULE_CONTEXT_INIT(&mail_user_module_register); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts-solr/fts-solr-plugin.c --- a/src/plugins/fts-solr/fts-solr-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -8,7 +8,7 @@ #include -const char *fts_solr_plugin_version = DOVECOT_VERSION; +const char *fts_solr_plugin_version = DOVECOT_ABI_VERSION; struct fts_solr_user_module fts_solr_user_module = MODULE_CONTEXT_INIT(&mail_user_module_register); diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts-squat/fts-squat-plugin.c --- a/src/plugins/fts-squat/fts-squat-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts-squat/fts-squat-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -3,7 +3,7 @@ #include "lib.h" #include "fts-squat-plugin.h" -const char *fts_squat_plugin_version = DOVECOT_VERSION; +const char *fts_squat_plugin_version = DOVECOT_ABI_VERSION; void fts_squat_plugin_init(struct module *module ATTR_UNUSED) { diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts/doveadm-fts.c --- a/src/plugins/fts/doveadm-fts.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts/doveadm-fts.c Fri Oct 12 00:51:41 2012 +0300 @@ -6,7 +6,7 @@ #include "doveadm-mail.h" #include "doveadm-fts.h" -const char *doveadm_fts_plugin_version = DOVECOT_VERSION; +const char *doveadm_fts_plugin_version = DOVECOT_ABI_VERSION; static int fts_namespace_find(struct mail_user *user, const char *ns_prefix, diff -r ce57bacc3010 -r 73916b7be94e src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Fri Oct 12 00:22:19 2012 +0300 +++ b/src/plugins/fts/fts-plugin.c Fri Oct 12 00:51:41 2012 +0300 @@ -7,7 +7,7 @@ #include -const char *fts_plugin_version = DOVECOT_VERSION; +const char *fts_plugin_version = DOVECOT_ABI_VERSION; From dovecot at dovecot.org Fri Oct 12 01:06:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 01:06:23 +0300 Subject: dovecot-2.2: lib-index: Error handling fix to cache transaction ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/70ebdddbdf9c changeset: 15200:70ebdddbdf9c user: Timo Sirainen date: Fri Oct 12 01:03:49 2012 +0300 description: lib-index: Error handling fix to cache transaction flushing. diffstat: src/lib-index/mail-cache-transaction.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 73916b7be94e -r 70ebdddbdf9c src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Fri Oct 12 00:51:41 2012 +0300 +++ b/src/lib-index/mail-cache-transaction.c Fri Oct 12 01:03:49 2012 +0300 @@ -355,7 +355,7 @@ ctx->last_rec_pos = 0; array_clear(&ctx->cache_data_seq); - return 0; + return ret; } static void From dovecot at dovecot.org Fri Oct 12 01:06:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 01:06:23 +0300 Subject: dovecot-2.2: i_set_failure_prefix() now takes printf-like parame... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d00cf8e0dae7 changeset: 15201:d00cf8e0dae7 user: Timo Sirainen date: Fri Oct 12 01:06:13 2012 +0300 description: i_set_failure_prefix() now takes printf-like parameters. This avoids using t_strdup_printf() with it, which was done very commonly. diffstat: src/doveadm/doveadm-mail.c | 2 +- src/doveadm/dsync/doveadm-dsync.c | 8 +++----- src/imap-urlauth/imap-urlauth-client.c | 3 +-- src/imap-urlauth/imap-urlauth-worker.c | 17 ++++++++--------- src/lib-master/master-login.c | 7 +++---- src/lib-master/master-service.c | 16 +++++++--------- src/lib-storage/mail-storage-service.c | 4 ++-- src/lib/failures.c | 19 +++++++++++++++---- src/lib/failures.h | 4 +++- src/lmtp/commands.c | 5 ++--- src/log/log-connection.c | 2 +- src/master/main.c | 2 +- 12 files changed, 47 insertions(+), 42 deletions(-) diffs (294 lines): diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/doveadm/doveadm-mail.c Fri Oct 12 01:06:13 2012 +0300 @@ -276,7 +276,7 @@ const char *error; int ret; - i_set_failure_prefix(t_strdup_printf("doveadm(%s): ", input->username)); + i_set_failure_prefix("doveadm(%s): ", input->username); /* see if we want to execute this command via (another) doveadm server */ diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Fri Oct 12 01:06:13 2012 +0300 @@ -238,7 +238,7 @@ i_assert(ctx->local_location != NULL); - i_set_failure_prefix(t_strdup_printf("dsync(%s): ", user->username)); + i_set_failure_prefix("dsync(%s): ", user->username); /* update mail_location and create another user for the second location. */ @@ -288,8 +288,7 @@ static void cmd_dsync_run_remote(struct mail_user *user) { - i_set_failure_prefix(t_strdup_printf("dsync-local(%s): ", - user->username)); + i_set_failure_prefix("dsync-local(%s): ", user->username); io_loop_run(current_ioloop); } @@ -515,8 +514,7 @@ user->admin = TRUE; user->dsyncing = TRUE; - i_set_failure_prefix(t_strdup_printf("dsync-remote(%s): ", - user->username)); + i_set_failure_prefix("dsync-remote(%s): ", user->username); temp_prefix = t_str_new(64); mail_user_set_get_temp_prefix(temp_prefix, user->set); diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/imap-urlauth/imap-urlauth-client.c --- a/src/imap-urlauth/imap-urlauth-client.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-client.c Fri Oct 12 01:06:13 2012 +0300 @@ -324,8 +324,7 @@ void client_destroy(struct client *client, const char *reason) { - i_set_failure_prefix(t_strdup_printf("%s: ", - master_service_get_name(master_service))); + i_set_failure_prefix("%s: ", master_service_get_name(master_service)); if (!client->disconnected) { if (reason == NULL) diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/imap-urlauth/imap-urlauth-worker.c --- a/src/imap-urlauth/imap-urlauth-worker.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker.c Fri Oct 12 01:06:13 2012 +0300 @@ -203,9 +203,8 @@ imap_urlauth_worker_client_count++; DLLIST_PREPEND(&imap_urlauth_worker_clients, client); - i_set_failure_prefix(t_strdup_printf("imap-urlauth[%s](%s): ", my_pid, - client->access_user)); - + i_set_failure_prefix("imap-urlauth[%s](%s): ", + my_pid, client->access_user); return client; } @@ -219,8 +218,8 @@ { char **app; - i_set_failure_prefix(t_strdup_printf("imap-urlauth[%s](%s): ", - my_pid, client->access_user)); + i_set_failure_prefix("imap-urlauth[%s](%s): ", + my_pid, client->access_user); if (client->url != NULL) { /* deinitialize url */ @@ -664,8 +663,8 @@ mail_user->username, client->access_user); } - i_set_failure_prefix(t_strdup_printf("imap-urlauth[%s](%s->%s): ", - my_pid, client->access_user, mail_user->username)); + i_set_failure_prefix("imap-urlauth[%s](%s->%s): ", + my_pid, client->access_user, mail_user->username); client_send_line(client, "OK"); return 1; @@ -889,8 +888,8 @@ client->access_user = i_strdup("anonymous"); client->access_anonymous = TRUE; } - i_set_failure_prefix(t_strdup_printf("imap-urlauth[%s](%s): ", my_pid, - client->access_user)); + i_set_failure_prefix("imap-urlauth[%s](%s): ", + my_pid, client->access_user); args++; while (*args != NULL) { diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lib-master/master-login.c --- a/src/lib-master/master-login.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lib-master/master-login.c Fri Oct 12 01:06:13 2012 +0300 @@ -182,8 +182,7 @@ i_error("close(fd_read client) failed: %m"); /* this client failed (login callback wasn't called). reset prefix to default. */ - i_set_failure_prefix(t_strdup_printf("%s: ", - client->conn->login->service->name)); + i_set_failure_prefix("%s: ", client->conn->login->service->name); } /* FIXME: currently we create a separate connection for each request, @@ -371,8 +370,8 @@ master_login_client_free(&client); return; } - i_set_failure_prefix(t_strdup_printf("%s(%s): ", - client->conn->login->service->name, auth_args[0])); + i_set_failure_prefix("%s(%s): ", client->conn->login->service->name, + auth_args[0]); if (conn->login->postlogin_socket_path == NULL) master_login_auth_finish(client, auth_args); diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lib-master/master-service.c Fri Oct 12 01:06:13 2012 +0300 @@ -133,7 +133,7 @@ lib_init(); /* Set a logging prefix temporarily. This will be ignored once the log is properly initialized */ - i_set_failure_prefix(t_strdup_printf("%s(init): ", name)); + i_set_failure_prefix("%s(init): ", name); /* ignore these signals as early as possible */ lib_signals_ignore(SIGPIPE, TRUE); @@ -189,12 +189,10 @@ we want to log */ if (getenv("LOG_SERVICE") != NULL) i_set_failure_internal(); - if (getenv("USER") != NULL) { - i_set_failure_prefix(t_strdup_printf("%s(%s): ", - name, getenv("USER"))); - } else { - i_set_failure_prefix(t_strdup_printf("%s: ", name)); - } + if (getenv("USER") != NULL) + i_set_failure_prefix("%s(%s): ", name, getenv("USER")); + else + i_set_failure_prefix("%s: ", name); if ((flags & MASTER_SERVICE_FLAG_STANDALONE) == 0) { /* initialize master_status structure */ @@ -262,7 +260,7 @@ if (getenv("LOG_SERVICE") != NULL && !service->log_directly) { /* logging via log service */ i_set_failure_internal(); - i_set_failure_prefix(prefix); + i_set_failure_prefix("%s", prefix); return; } @@ -287,7 +285,7 @@ &facility)) facility = LOG_MAIL; i_set_failure_syslog("dovecot", LOG_NDELAY, facility); - i_set_failure_prefix(prefix); + i_set_failure_prefix("%s", prefix); if (strcmp(service->set->log_path, "syslog") != 0) { /* set error handlers back to file */ diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lib-storage/mail-storage-service.c Fri Oct 12 01:06:13 2012 +0300 @@ -659,14 +659,14 @@ { struct mail_storage_service_user *user = context; - i_set_failure_prefix(user->log_prefix); + i_set_failure_prefix("%s", user->log_prefix); } static void mail_storage_service_io_deactivate(void *context) { struct mail_storage_service_user *user = context; - i_set_failure_prefix(user->service_ctx->default_log_prefix); + i_set_failure_prefix("%s", user->service_ctx->default_log_prefix); } static void diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lib/failures.c --- a/src/lib/failures.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lib/failures.c Fri Oct 12 01:06:13 2012 +0300 @@ -477,7 +477,7 @@ void i_set_failure_file(const char *path, const char *prefix) { - i_set_failure_prefix(prefix); + i_set_failure_prefix("%s", prefix); if (log_info_fd != STDERR_FILENO && log_info_fd != log_fd) { if (close(log_info_fd) < 0) @@ -514,11 +514,22 @@ (void)write_full(2, str, strlen(str)); } -void i_set_failure_prefix(const char *prefix) +void i_set_failure_prefix(const char *prefix_fmt, ...) +{ + va_list args; + + va_start(args, prefix_fmt); + i_free(log_prefix); + log_prefix = i_strdup_vprintf(prefix_fmt, args); + va_end(args); + + log_prefix_sent = FALSE; +} + +void i_unset_failure_prefix(void) { i_free(log_prefix); - log_prefix = i_strdup(prefix); - + log_prefix = i_strdup(""); log_prefix_sent = FALSE; } diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lib/failures.h --- a/src/lib/failures.h Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lib/failures.h Fri Oct 12 01:06:13 2012 +0300 @@ -111,7 +111,9 @@ void i_set_debug_file(const char *path); /* Set the failure prefix. */ -void i_set_failure_prefix(const char *prefix); +void i_set_failure_prefix(const char *prefix_fmt, ...) ATTR_FORMAT(1, 2); +/* Set prefix to "". */ +void i_unset_failure_prefix(void); /* Prefix failures with a timestamp. fmt is in strftime() format. */ void i_set_failure_timestamp_format(const char *fmt); /* When logging with internal error protocol, update the process's current diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/lmtp/commands.c --- a/src/lmtp/commands.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/lmtp/commands.c Fri Oct 12 01:06:13 2012 +0300 @@ -642,8 +642,7 @@ i_unreached(); } - i_set_failure_prefix(t_strdup_printf("lmtp(%s, %s): ", - my_pid, username)); + i_set_failure_prefix("lmtp(%s, %s): ", my_pid, username); if (mail_storage_service_next(storage_service, rcpt->service_user, &client->state.dest_user) < 0) { client_send_line(client, ERRSTR_TEMP_MAILBOX_FAIL, @@ -720,7 +719,7 @@ while (client->state.rcpt_idx < count) { ret = client_deliver(client, &rcpts[client->state.rcpt_idx], src_mail, session); - i_set_failure_prefix(t_strdup_printf("lmtp(%s): ", my_pid)); + i_set_failure_prefix("lmtp(%s): ", my_pid); client->state.rcpt_idx++; if (ret == 0) diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/log/log-connection.c --- a/src/log/log-connection.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/log/log-connection.c Fri Oct 12 01:06:13 2012 +0300 @@ -104,7 +104,7 @@ log_error_buffer_add(log->errorbuf, &err); break; } - i_set_failure_prefix(prefix); + i_set_failure_prefix("%s", prefix); i_log_type(ctx, "%s", text); i_set_failure_prefix("log: "); } diff -r 70ebdddbdf9c -r d00cf8e0dae7 src/master/main.c --- a/src/master/main.c Fri Oct 12 01:03:49 2012 +0300 +++ b/src/master/main.c Fri Oct 12 01:06:13 2012 +0300 @@ -770,7 +770,7 @@ MASTER_SERVICE_FLAG_STANDALONE | MASTER_SERVICE_FLAG_DONT_LOG_TO_STDERR, &argc, &argv, "+Fanp"); - i_set_failure_prefix(""); + i_unset_failure_prefix(); io_loop_set_time_moved_callback(current_ioloop, master_time_moved); From dovecot at dovecot.org Fri Oct 12 01:17:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 01:17:23 +0300 Subject: dovecot-2.2: hostpid_init(): Don't waste memory from data stack. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a446d8bc2be5 changeset: 15202:a446d8bc2be5 user: Timo Sirainen date: Fri Oct 12 01:17:05 2012 +0300 description: hostpid_init(): Don't waste memory from data stack. Even though it's only a few bytes, it was being wasted permanently for all processes. diffstat: src/lib/hostpid.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r d00cf8e0dae7 -r a446d8bc2be5 src/lib/hostpid.c --- a/src/lib/hostpid.c Fri Oct 12 01:06:13 2012 +0300 +++ b/src/lib/hostpid.c Fri Oct 12 01:17:05 2012 +0300 @@ -28,8 +28,7 @@ /* allow calling hostpid_init() multiple times to reset hostname */ i_free_and_null(my_domain); - if (i_strocpy(pid, dec2str(getpid()), sizeof(pid)) < 0) - i_unreached(); + i_snprintf(pid, sizeof(pid), "%lld", (long long)getpid()); my_pid = pid; } From dovecot at dovecot.org Fri Oct 12 01:21:38 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 01:21:38 +0300 Subject: dovecot-2.2: imap: Don't waste data stack memory at startup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b25fb761e580 changeset: 15203:b25fb761e580 user: Timo Sirainen date: Fri Oct 12 01:21:30 2012 +0300 description: imap: Don't waste data stack memory at startup. diffstat: src/imap/main.c | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (37 lines): diff -r a446d8bc2be5 -r b25fb761e580 src/imap/main.c --- a/src/imap/main.c Fri Oct 12 01:17:05 2012 +0300 +++ b/src/imap/main.c Fri Oct 12 01:21:30 2012 +0300 @@ -362,12 +362,6 @@ } } - login_set.auth_socket_path = t_abspath("auth-master"); - if (argv[optind] != NULL) - login_set.postlogin_socket_path = t_abspath(argv[optind]); - login_set.callback = login_client_connected; - login_set.failure_callback = login_client_failed; - master_service_init_finish(master_service); master_service_set_die_callback(master_service, imap_die); @@ -387,10 +381,18 @@ T_BEGIN { main_stdio_run(username); } T_END; - } else { + } else T_BEGIN { + login_set.auth_socket_path = t_abspath("auth-master"); + if (argv[optind] != NULL) { + login_set.postlogin_socket_path = + t_abspath(argv[optind]); + } + login_set.callback = login_client_connected; + login_set.failure_callback = login_client_failed; + master_login = master_login_init(master_service, &login_set); io_loop_set_running(current_ioloop); - } + } T_END; if (io_loop_is_running(current_ioloop)) master_service_run(master_service, client_connected); From dovecot at dovecot.org Fri Oct 12 03:08:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:08:36 +0300 Subject: dovecot-2.2: config: Don't include lib-master/*.c files in all-s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c30673b5ec1b changeset: 15204:c30673b5ec1b user: Timo Sirainen date: Fri Oct 12 02:59:59 2012 +0300 description: config: Don't include lib-master/*.c files in all-settings.c lib-master is already being linked to the binaries anyway. diffstat: configure.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b25fb761e580 -r c30673b5ec1b configure.in --- a/configure.in Fri Oct 12 01:21:30 2012 +0300 +++ b/configure.in Fri Oct 12 02:59:59 2012 +0300 @@ -2699,7 +2699,7 @@ dnl ** dnl get a list of setting .[ch] files, but list .h files first -FILES1=`find $srcdir/src -name '*settings.[[ch]]'|grep "$srcdir/src/lib-" | sed 's/^\(.*\)\(.\)$/\2 \1\2/' | sort -r | sed s/^..//` +FILES1=`find $srcdir/src -name '*settings.[[ch]]'|grep "$srcdir/src/lib-" | sed 's/^\(.*\)\(.\)$/\2 \1\2/' | grep -v 'lib-master.*c$' | sort -r | sed s/^..//` FILES2=`find $srcdir/src -name '*settings.[[ch]]'|grep -v "$srcdir/src/lib-" | sed 's/^\(.*\)\(.\)$/\2 \1\2/' | grep -v all-settings | sort -r | sed s/^..//` SETTING_FILES=`echo $FILES1 $FILES2 | sed -e s,$srcdir/src,./src,g -e 's,./src,$(top_srcdir)/src,g'` AC_SUBST(SETTING_FILES) From dovecot at dovecot.org Fri Oct 12 03:08:37 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:08:37 +0300 Subject: dovecot-2.2: --without-shared-libs: Link Dovecot libraries with ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0c7d2f8a185 changeset: 15205:b0c7d2f8a185 user: Timo Sirainen date: Fri Oct 12 03:08:13 2012 +0300 description: --without-shared-libs: Link Dovecot libraries with --whole-archive flag for binaries. This removes the need for the ugly unused_objects lists for binaries, which were needed to avoid plugins from failing because they were missing some functions. Apparently there's no easy way to use --whole-archive properly with libtool, so there's now a rather ugly cc-wrapper.sh that does it. Also this is done only when GNU ld is deted. Most people are going to use --with-shared-libs though, so I don't think any of this is going to be a real problem. diffstat: .hgignore | 1 + cc-wrapper.sh.in | 8 ++++++++ configure.in | 9 +++++++++ src/doveadm/Makefile.am | 9 +-------- src/indexer/Makefile.am | 8 -------- src/lda/Makefile.am | 15 ++------------- src/lmtp/Makefile.am | 17 ++--------------- src/pop3/Makefile.am | 8 -------- 8 files changed, 23 insertions(+), 52 deletions(-) diffs (175 lines): diff -r c30673b5ec1b -r b0c7d2f8a185 .hgignore --- a/.hgignore Fri Oct 12 02:59:59 2012 +0300 +++ b/.hgignore Fri Oct 12 03:08:13 2012 +0300 @@ -1,6 +1,7 @@ syntax: glob aclocal.m4 autom4te.cache +cc-wrapper.sh compile config.cache config.guess diff -r c30673b5ec1b -r b0c7d2f8a185 cc-wrapper.sh.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cc-wrapper.sh.in Fri Oct 12 03:08:13 2012 +0300 @@ -0,0 +1,8 @@ +#!/bin/sh + +if echo "$*" | grep -- -ldl > /dev/null; then + # the binary uses plugins. make sure we include everything from .a libs + exec @CC@ -Wl,--whole-archive $* -Wl,--no-whole-archive +else + exec @CC@ $* +fi diff -r c30673b5ec1b -r b0c7d2f8a185 configure.in --- a/configure.in Fri Oct 12 02:59:59 2012 +0300 +++ b/configure.in Fri Oct 12 03:08:13 2012 +0300 @@ -2723,6 +2723,15 @@ LDFLAGS="\$(NOPLUGIN_LDFLAGS) $LDFLAGS" AC_SUBST(NOPLUGIN_LDFLAGS) +if test "$with_gnu_ld" = yes -a "$want_shared_libs" = "no"; then + # libtool can't handle using whole-archive flags, so we need to do this + # with a CC wrapper.. shouldn't be much of a problem, since most people + # are building with shared libs. + sed "s/@CC@/$CC/" < $srcdir/cc-wrapper.sh.in > cc-wrapper.sh + chmod +x cc-wrapper.sh + CC=`pwd`/cc-wrapper.sh +fi + if test "$docdir" = ""; then dnl docdir supported only by autoconf v2.59c and later docdir='${datadir}/doc/${PACKAGE_TARNAME}' diff -r c30673b5ec1b -r b0c7d2f8a185 src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Fri Oct 12 02:59:59 2012 +0300 +++ b/src/doveadm/Makefile.am Fri Oct 12 03:08:13 2012 +0300 @@ -26,20 +26,13 @@ -DBINDIR=\""$(bindir)"\" \ -DMANDIR=\""$(mandir)"\" -if !BUILD_SHARED_LIBS -unused_objects = \ - ../lib-imap/imap-util.o \ - ../lib-storage/mail-search-parser-imap.o -endif - cmd_pw_libs = \ ../auth/libpassword.a \ ../lib-ntlm/libntlm.a \ ../lib-otp/libotp.a libs = \ - dsync/libdsync.la \ - $(unused_objects) + dsync/libdsync.la doveadm_LDADD = \ $(libs) \ diff -r c30673b5ec1b -r b0c7d2f8a185 src/indexer/Makefile.am --- a/src/indexer/Makefile.am Fri Oct 12 02:59:59 2012 +0300 +++ b/src/indexer/Makefile.am Fri Oct 12 03:08:13 2012 +0300 @@ -21,19 +21,11 @@ worker-connection.c \ worker-pool.c -if !BUILD_SHARED_LIBS -unused_objects = \ - ../lib-imap/imap-util.o \ - ../lib-storage/mail-search-parser-imap.o -endif - indexer_worker_LDADD = \ - $(unused_objects) \ $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) indexer_worker_DEPENDENCIES = \ - $(unused_objects) \ $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) indexer_worker_SOURCES = \ diff -r c30673b5ec1b -r b0c7d2f8a185 src/lda/Makefile.am --- a/src/lda/Makefile.am Fri Oct 12 02:59:59 2012 +0300 +++ b/src/lda/Makefile.am Fri Oct 12 03:08:13 2012 +0300 @@ -16,24 +16,13 @@ dovecot_lda_LDFLAGS = -export-dynamic -if !BUILD_SHARED_LIBS -unused_objects = \ - ../lib-mail/message-header-encode.o \ - ../lib-imap/imap-util.o \ - ../lib-storage/mail-search-parser-imap.o -endif - -libs = \ - $(unused_objects) \ - $(LIBDOVECOT_LDA) - dovecot_lda_LDADD = \ - $(libs) \ + $(LIBDOVECOT_LDA) \ $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) dovecot_lda_DEPENDENCIES = \ - $(libs) \ + $(LIBDOVECOT_LDA) \ $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) diff -r c30673b5ec1b -r b0c7d2f8a185 src/lmtp/Makefile.am --- a/src/lmtp/Makefile.am Fri Oct 12 02:59:59 2012 +0300 +++ b/src/lmtp/Makefile.am Fri Oct 12 03:08:13 2012 +0300 @@ -17,26 +17,13 @@ lmtp_LDFLAGS = -export-dynamic -if !BUILD_SHARED_LIBS -unused_objects = \ - ../lib-mail/message-header-encode.o \ - ../lib-imap/imap-util.o \ - ../lib-storage/mail-search-parser-imap.o \ - ../lib-lda/smtp-client.o \ - ../lib-lda/mail-send.o -endif - -libs = \ - $(unused_objects) \ - $(LIBDOVECOT_LDA) - lmtp_LDADD = \ - $(libs) \ + $(LIBDOVECOT_LDA) \ $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) lmtp_DEPENDENCIES = \ - $(libs) \ + $(LIBDOVECOT_LDA) \ $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) diff -r c30673b5ec1b -r b0c7d2f8a185 src/pop3/Makefile.am --- a/src/pop3/Makefile.am Fri Oct 12 02:59:59 2012 +0300 +++ b/src/pop3/Makefile.am Fri Oct 12 03:08:13 2012 +0300 @@ -12,19 +12,11 @@ pop3_LDFLAGS = -export-dynamic -if !BUILD_SHARED_LIBS -unused_objects = \ - ../lib-imap/imap-util.o \ - ../lib-storage/mail-search-parser-imap.o -endif - pop3_LDADD = \ - $(unused_objects) \ $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) pop3_DEPENDENCIES = \ - $(unused_objects) \ $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) From dovecot at dovecot.org Fri Oct 12 03:09:51 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:09:51 +0300 Subject: dovecot-2.2: Makefile: Added cc-wrapper.sh.in missing from last ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dbf88dc1b873 changeset: 15206:dbf88dc1b873 user: Timo Sirainen date: Fri Oct 12 03:09:27 2012 +0300 description: Makefile: Added cc-wrapper.sh.in missing from last commit diffstat: Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r b0c7d2f8a185 -r dbf88dc1b873 Makefile.am --- a/Makefile.am Fri Oct 12 03:08:13 2012 +0300 +++ b/Makefile.am Fri Oct 12 03:09:27 2012 +0300 @@ -14,6 +14,7 @@ ChangeLog \ is-tagged.py \ run-test.sh \ + cc-wrapper.sh.in \ update-version.sh \ $(conf_DATA) From dovecot at dovecot.org Fri Oct 12 03:18:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:18:12 +0300 Subject: dovecot-2.2: Build imapc and pop3c always as plugins. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/269104a0821b changeset: 15208:269104a0821b user: Timo Sirainen date: Fri Oct 12 03:17:56 2012 +0300 description: Build imapc and pop3c always as plugins. This avoids having to link openssl library to all binaries that use lib-storage. It appears that simply by linking with openssl causes each such process to use 100-200 kB of more memory. With 10k imap processes this is 1-2 GB of wasted memory. The imapc and pop3c are still registered as stubs, and their plugins are automatically loaded when needed. diffstat: configure.in | 25 +++++--------------- src/lib-storage/index/imapc/Makefile.am | 17 +++++++++++++- src/lib-storage/index/imapc/imapc-list.c | 2 - src/lib-storage/index/imapc/imapc-list.h | 2 + src/lib-storage/index/imapc/imapc-plugin.c | 21 +++++++++++++++++ src/lib-storage/index/imapc/imapc-plugin.h | 7 +++++ src/lib-storage/index/imapc/imapc-setting-storage.c | 22 ++++++++++++++++++ src/lib-storage/index/imapc/imapc-settings.h | 2 + src/lib-storage/index/imapc/imapc-storage.c | 3 -- src/lib-storage/index/imapc/imapc-storage.h | 3 ++ src/lib-storage/index/pop3c/Makefile.am | 16 ++++++++++++- src/lib-storage/index/pop3c/pop3c-plugin.c | 19 +++++++++++++++ src/lib-storage/index/pop3c/pop3c-plugin.h | 7 +++++ src/lib-storage/index/pop3c/pop3c-setting-storage.c | 22 ++++++++++++++++++ src/lib-storage/index/pop3c/pop3c-settings.h | 2 + src/lib-storage/index/pop3c/pop3c-storage.c | 3 -- src/lib-storage/index/pop3c/pop3c-storage.h | 2 + 17 files changed, 146 insertions(+), 29 deletions(-) diffs (truncated from 370 to 300 lines): diff -r 985ebfb17f35 -r 269104a0821b configure.in --- a/configure.in Fri Oct 12 03:12:19 2012 +0300 +++ b/configure.in Fri Oct 12 03:17:56 2012 +0300 @@ -243,14 +243,14 @@ want_gc=no) AC_ARG_WITH(storages, -AS_HELP_STRING([--with-storages], [Build with specified mail storage formats (mdbox sdbox maildir mbox cydir imapc pop3c)]), [ +AS_HELP_STRING([--with-storages], [Build with specified mail storage formats (mdbox sdbox maildir mbox cydir)]), [ if test "$withval" = "yes" || test "$withval" = "no"; then AC_MSG_ERROR([--with-storages needs storage list as parameter]) fi mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ], - mail_storages="shared mdbox sdbox maildir mbox cydir imapc pop3c") + mail_storages="shared mdbox sdbox maildir mbox cydir") AC_SUBST(mail_storages) -mail_storages="$mail_storages raw" +mail_storages="$mail_storages imapc_stub pop3c_stub raw" # drop duplicates duplicates=`(for i in $mail_storages; do echo $i; done)|sort|uniq -d|xargs echo` if test "$duplicates" != ""; then @@ -2454,8 +2454,8 @@ sdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-single/libstorage_dbox_single.la' mdbox_libs='$(top_builddir)/src/lib-storage/index/dbox-multi/libstorage_dbox_multi.la' cydir_libs='$(top_builddir)/src/lib-storage/index/cydir/libstorage_cydir.la' -imapc_libs='$(top_builddir)/src/lib-storage/index/imapc/libstorage_imapc.la $(top_builddir)/src/lib-imap-client/libimap_client.la' -pop3c_libs='$(top_builddir)/src/lib-storage/index/pop3c/libstorage_pop3c.la' +imapc_stub_libs='$(top_builddir)/src/lib-storage/index/imapc/libstorage_imapc.la' +pop3c_stub_libs='$(top_builddir)/src/lib-storage/index/pop3c/libstorage_pop3c.la' raw_libs='$(top_builddir)/src/lib-storage/index/raw/libstorage_raw.la' shared_libs='$(top_builddir)/src/lib-storage/index/shared/libstorage_shared.la' @@ -2475,21 +2475,8 @@ LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS $dbox_common_libs" dbox_common_libs="" fi - if test $storage = imapc; then - mailbox_list_drivers="$mailbox_list_drivers imapc" - want_ssl_libs=yes - fi - if test $storage = pop3c; then - want_ssl_libs=yes - fi done -LINKED_STORAGE_LDADD= -if test "$want_ssl_libs" = yes; then - LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS \$(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la" - LINKED_STORAGE_LDADD="$SSL_LIBS" -fi AC_SUBST(LINKED_STORAGE_LIBS) -AC_SUBST(LINKED_STORAGE_LDADD) AC_SUBST(mailbox_list_drivers) AC_DEFINE_UNQUOTED(MAIL_STORAGES, "$mail_storages", List of compiled in mail storages) @@ -2521,7 +2508,7 @@ LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi -LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS $LINKED_STORAGE_LDADD" +LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS" LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la' AC_SUBST(LIBDOVECOT) AC_SUBST(LIBDOVECOT_DEPS) diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/Makefile.am --- a/src/lib-storage/index/imapc/Makefile.am Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/Makefile.am Fri Oct 12 03:17:56 2012 +0300 @@ -1,4 +1,5 @@ noinst_LTLIBRARIES = libstorage_imapc.la +module_LTLIBRARIES = lib20_imapc_plugin.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -12,16 +13,27 @@ -I$(top_srcdir)/src/lib-storage/list \ -I$(top_srcdir)/src/lib-storage/index +NOPLUGIN_LDFLAGS = +lib20_imapc_plugin_la_LDFLAGS = -module -avoid-version + libstorage_imapc_la_SOURCES = \ + imapc-settings.c \ + imapc-setting-storage.c + +lib20_imapc_plugin_la_SOURCES = \ imapc-list.c \ imapc-mail.c \ imapc-mail-fetch.c \ imapc-mailbox.c \ + imapc-plugin.c \ imapc-save.c \ - imapc-settings.c \ imapc-sync.c \ imapc-storage.c +lib20_imapc_plugin_la_LIBADD = \ + ../../../lib-imap-client/libimap_client.la \ + ../../../lib-ssl-iostream/libssl_iostream.la + headers = \ imapc-list.h \ imapc-mail.h \ @@ -31,3 +43,6 @@ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) + +noinst_HEADERS = \ + imapc-plugin.h diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 03:17:56 2012 +0300 @@ -20,8 +20,6 @@ struct mailbox_info info; }; -extern struct mailbox_list imapc_mailbox_list; - static struct mailbox_list *imapc_list_alloc(void) { struct imapc_mailbox_list *list; diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-list.h --- a/src/lib-storage/index/imapc/imapc-list.h Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.h Fri Oct 12 03:17:56 2012 +0300 @@ -22,6 +22,8 @@ unsigned int index_list_failed:1; }; +extern struct mailbox_list imapc_mailbox_list; + int imapc_list_get_mailbox_flags(struct mailbox_list *list, const char *name, enum mailbox_info_flags *flags_r); diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-plugin.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/imapc/imapc-plugin.c Fri Oct 12 03:17:56 2012 +0300 @@ -0,0 +1,21 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "imapc-list.h" +#include "imapc-storage.h" +#include "imapc-plugin.h" + +const char *imapc_plugin_version = DOVECOT_ABI_VERSION; + +void imapc_plugin_init(struct module *module ATTR_UNUSED) +{ + mail_storage_class_unregister(&imapc_stub_storage); + mail_storage_class_register(&imapc_storage); + mailbox_list_register(&imapc_mailbox_list); +} + +void imapc_plugin_deinit(void) +{ + mail_storage_class_unregister(&imapc_storage); + mailbox_list_unregister(&imapc_mailbox_list); +} diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-plugin.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/imapc/imapc-plugin.h Fri Oct 12 03:17:56 2012 +0300 @@ -0,0 +1,7 @@ +#ifndef IMAPC_PLUGIN_H +#define IMAPC_PLUGIN_H + +void imapc_plugin_init(struct module *module); +void imapc_plugin_deinit(void); + +#endif diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-setting-storage.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/imapc/imapc-setting-storage.c Fri Oct 12 03:17:56 2012 +0300 @@ -0,0 +1,22 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "imapc-storage.h" +#include "imapc-settings.h" + +struct mail_storage imapc_stub_storage = { + .name = IMAPC_STORAGE_NAME, + .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT, + + .v = { + imapc_get_setting_parser_info, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + } +}; diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-settings.h --- a/src/lib-storage/index/imapc/imapc-settings.h Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-settings.h Fri Oct 12 03:17:56 2012 +0300 @@ -30,6 +30,8 @@ enum imapc_features parsed_features; }; +extern struct mail_storage imapc_stub_storage; + const struct setting_parser_info *imapc_get_setting_parser_info(void); #endif diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Fri Oct 12 03:17:56 2012 +0300 @@ -27,9 +27,6 @@ enum mail_error error; }; -extern struct mail_storage imapc_storage; -extern struct mailbox imapc_mailbox; - static struct imapc_resp_code_map imapc_resp_code_map[] = { { IMAP_RESP_CODE_UNAVAILABLE, MAIL_ERROR_TEMP }, { IMAP_RESP_CODE_AUTHFAILED, MAIL_ERROR_PERM }, diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.h Fri Oct 12 03:17:56 2012 +0300 @@ -110,6 +110,9 @@ int ret; }; +extern struct mail_storage imapc_storage; +extern struct mailbox imapc_mailbox; + struct mail_save_context * imapc_save_alloc(struct mailbox_transaction_context *_t); int imapc_save_begin(struct mail_save_context *ctx, struct istream *input); diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/pop3c/Makefile.am --- a/src/lib-storage/index/pop3c/Makefile.am Fri Oct 12 03:12:19 2012 +0300 +++ b/src/lib-storage/index/pop3c/Makefile.am Fri Oct 12 03:17:56 2012 +0300 @@ -1,4 +1,5 @@ noinst_LTLIBRARIES = libstorage_pop3c.la +module_LTLIBRARIES = lib20_pop3c_plugin.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -11,13 +12,23 @@ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/lib-storage/index +NOPLUGIN_LDFLAGS = +lib20_pop3c_plugin_la_LDFLAGS = -module -avoid-version + libstorage_pop3c_la_SOURCES = \ + pop3c-settings.c \ + pop3c-setting-storage.c + +lib20_pop3c_plugin_la_SOURCES = \ pop3c-client.c \ pop3c-mail.c \ - pop3c-settings.c \ + pop3c-plugin.c \ pop3c-storage.c \ pop3c-sync.c +lib20_pop3c_plugin_la_LIBADD = \ + ../../../lib-ssl-iostream/libssl_iostream.la + headers = \ pop3c-client.h \ pop3c-settings.h \ @@ -26,3 +37,6 @@ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) + +noinst_HEADERS = \ + pop3c-plugin.h diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/pop3c/pop3c-plugin.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/pop3c/pop3c-plugin.c Fri Oct 12 03:17:56 2012 +0300 @@ -0,0 +1,19 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "pop3c-storage.h" +#include "pop3c-settings.h" +#include "pop3c-plugin.h" + +const char *pop3c_plugin_version = DOVECOT_ABI_VERSION; + +void pop3c_plugin_init(struct module *module ATTR_UNUSED) +{ + mail_storage_class_unregister(&pop3c_stub_storage); + mail_storage_class_register(&pop3c_storage); +} + +void pop3c_plugin_deinit(void) +{ + mail_storage_class_unregister(&pop3c_storage); +} diff -r 985ebfb17f35 -r 269104a0821b src/lib-storage/index/pop3c/pop3c-plugin.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/pop3c/pop3c-plugin.h Fri Oct 12 03:17:56 2012 +0300 @@ -0,0 +1,7 @@ From dovecot at dovecot.org Fri Oct 12 03:18:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:18:12 +0300 Subject: dovecot-2.2: lib-storage: Added mail_user_get_storage_class(), w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/985ebfb17f35 changeset: 15207:985ebfb17f35 user: Timo Sirainen date: Fri Oct 12 03:12:19 2012 +0300 description: lib-storage: Added mail_user_get_storage_class(), which auto-loads storage plugins. The auto-loading is done only if there's already a stub storage registered. diffstat: src/lib-storage/index/shared/shared-storage.c | 2 +- src/lib-storage/mail-storage.c | 2 +- src/lib-storage/mail-user.c | 54 +++++++++++++++++++++++++++ src/lib-storage/mail-user.h | 5 ++ 4 files changed, 61 insertions(+), 2 deletions(-) diffs (105 lines): diff -r dbf88dc1b873 -r 985ebfb17f35 src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Fri Oct 12 03:09:27 2012 +0300 +++ b/src/lib-storage/index/shared/shared-storage.c Fri Oct 12 03:12:19 2012 +0300 @@ -49,7 +49,7 @@ p_strdup(_storage->pool, ns->unexpanded_set->location); storage->storage_class_name = p_strdup(_storage->pool, driver); - storage_class = mail_storage_find_class(driver); + storage_class = mail_user_get_storage_class(_storage->user, driver); if (storage_class != NULL) _storage->class_flags = storage_class->class_flags; else if (strcmp(driver, "auto") != 0) { diff -r dbf88dc1b873 -r 985ebfb17f35 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Fri Oct 12 03:09:27 2012 +0300 +++ b/src/lib-storage/mail-storage.c Fri Oct 12 03:12:19 2012 +0300 @@ -144,7 +144,7 @@ list_set->root_dir = NULL; } } else { - storage_class = mail_storage_find_class(driver); + storage_class = mail_user_get_storage_class(ns->user, driver); if (storage_class == NULL) { *error_r = t_strdup_printf( "Unknown mail storage driver %s", driver); diff -r dbf88dc1b873 -r 985ebfb17f35 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Fri Oct 12 03:09:27 2012 +0300 +++ b/src/lib-storage/mail-user.c Fri Oct 12 03:12:19 2012 +0300 @@ -16,6 +16,7 @@ #include "mountpoint-list.h" #include "mail-storage-settings.h" #include "mail-storage-private.h" +#include "mail-storage-service.h" #include "mail-namespace.h" #include "mail-storage.h" #include "mail-user.h" @@ -430,3 +431,56 @@ } return TRUE; } + +static void +mail_user_try_load_class_plugin(struct mail_user *user, const char *name) +{ + struct module_dir_load_settings mod_set; + struct module *module; + unsigned int name_len = strlen(name); + + memset(&mod_set, 0, sizeof(mod_set)); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.binary_name = master_service_get_name(master_service); + mod_set.setting_name = ""; + mod_set.require_init_funcs = TRUE; + mod_set.debug = user->mail_debug; + + mail_storage_service_modules = + module_dir_load_missing(mail_storage_service_modules, + user->set->mail_plugin_dir, + name, &mod_set); + /* initialize the module (and only this module!) immediately so that + the class gets registered */ + for (module = mail_storage_service_modules; module != NULL; module = module->next) { + if (strncmp(module->name, name, name_len) == 0 && + strcmp(module->name + name_len, "_plugin") == 0) { + if (!module->initialized) { + module->initialized = TRUE; + module->init(module); + } + break; + } + } +} + +struct mail_storage * +mail_user_get_storage_class(struct mail_user *user, const char *name) +{ + struct mail_storage *storage; + + storage = mail_storage_find_class(name); + if (storage == NULL || storage->v.alloc != NULL) + return storage; + + /* it's implemented by a plugin. load it and check again. */ + mail_user_try_load_class_plugin(user, name); + + storage = mail_storage_find_class(name); + if (storage != NULL && storage->v.alloc == NULL) { + i_error("Storage driver '%s' exists as a stub, " + "but its plugin couldn't be loaded", name); + return NULL; + } + return storage; +} diff -r dbf88dc1b873 -r 985ebfb17f35 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Fri Oct 12 03:09:27 2012 +0300 +++ b/src/lib-storage/mail-user.h Fri Oct 12 03:12:19 2012 +0300 @@ -137,4 +137,9 @@ bool mail_user_is_path_mounted(struct mail_user *user, const char *path, const char **error_r); +/* Basically the same as mail_storage_find_class(), except automatically load + storage plugins when needed. */ +struct mail_storage * +mail_user_get_storage_class(struct mail_user *user, const char *name); + #endif From dovecot at dovecot.org Fri Oct 12 03:47:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 03:47:52 +0300 Subject: dovecot-2.2: lib-storage: Added optional mailbox_list.init(), wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a44be96f55c1 changeset: 15209:a44be96f55c1 user: Timo Sirainen date: Fri Oct 12 03:47:41 2012 +0300 description: lib-storage: Added optional mailbox_list.init(), which can check for failure. "index" backend now gives a nice error message instead of crashing when mailbox_list_index=no diffstat: src/lib-storage/index/imapc/imapc-list.c | 1 + src/lib-storage/index/shared/shared-list.c | 1 + src/lib-storage/list/mailbox-list-fs.c | 1 + src/lib-storage/list/mailbox-list-index-backend.c | 10 ++++++++++ src/lib-storage/list/mailbox-list-maildir.c | 2 ++ src/lib-storage/list/mailbox-list-none.c | 1 + src/lib-storage/mailbox-list-private.h | 1 + src/lib-storage/mailbox-list.c | 7 +++++++ 8 files changed, 24 insertions(+), 0 deletions(-) diffs (118 lines): diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 03:47:41 2012 +0300 @@ -664,6 +664,7 @@ { imapc_list_alloc, + NULL, imapc_list_deinit, NULL, imapc_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/index/shared/shared-list.c --- a/src/lib-storage/index/shared/shared-list.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/index/shared/shared-list.c Fri Oct 12 03:47:41 2012 +0300 @@ -281,6 +281,7 @@ { shared_list_alloc, + NULL, shared_list_deinit, shared_get_storage, shared_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/list/mailbox-list-fs.c --- a/src/lib-storage/list/mailbox-list-fs.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs.c Fri Oct 12 03:47:41 2012 +0300 @@ -481,6 +481,7 @@ { fs_list_alloc, + NULL, fs_list_deinit, NULL, fs_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Fri Oct 12 03:47:41 2012 +0300 @@ -34,6 +34,15 @@ return &list->list; } +static int index_list_init(struct mailbox_list *_list, const char **error_r) +{ + if (!_list->mail_set->mailbox_list_index) { + *error_r = "LAYOUT=index requires mailbox_list_index=yes"; + return -1; + } + return 0; +} + static void index_list_deinit(struct mailbox_list *_list) { struct index_mailbox_list *list = (struct index_mailbox_list *)_list; @@ -555,6 +564,7 @@ { index_list_alloc, + index_list_init, index_list_deinit, NULL, index_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/list/mailbox-list-maildir.c --- a/src/lib-storage/list/mailbox-list-maildir.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-maildir.c Fri Oct 12 03:47:41 2012 +0300 @@ -476,6 +476,7 @@ { maildir_list_alloc, + NULL, maildir_list_deinit, NULL, maildir_list_get_hierarchy_sep, @@ -508,6 +509,7 @@ { imapdir_list_alloc, + NULL, maildir_list_deinit, NULL, maildir_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/list/mailbox-list-none.c --- a/src/lib-storage/list/mailbox-list-none.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-none.c Fri Oct 12 03:47:41 2012 +0300 @@ -158,6 +158,7 @@ { none_list_alloc, + NULL, none_list_deinit, NULL, none_list_get_hierarchy_sep, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/mailbox-list-private.h Fri Oct 12 03:47:41 2012 +0300 @@ -27,6 +27,7 @@ struct mailbox_list_vfuncs { struct mailbox_list *(*alloc)(void); + int (*init)(struct mailbox_list *list, const char **error_r); void (*deinit)(struct mailbox_list *list); int (*get_storage)(struct mailbox_list **list, const char *vname, diff -r 269104a0821b -r a44be96f55c1 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Fri Oct 12 03:17:56 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Fri Oct 12 03:47:41 2012 +0300 @@ -179,6 +179,13 @@ } list->set.utf8 = set->utf8; + if (list->v.init != NULL) { + if (list->v.init(list, error_r) < 0) { + list->v.deinit(list); + return -1; + } + } + if (ns->mail_set->mail_debug) { i_debug("%s: root=%s, index=%s, indexpvt=%s, control=%s, inbox=%s, alt=%s", list->name, From dovecot at dovecot.org Fri Oct 12 05:46:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 05:46:17 +0300 Subject: dovecot-2.2: lib-storage: Added support for "broken_char" settin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22e7322e8601 changeset: 15210:22e7322e8601 user: Timo Sirainen date: Fri Oct 12 05:45:42 2012 +0300 description: lib-storage: Added support for "broken_char" setting for escaping invalid mailbox names. This is currently enabled only for imapc backend, where '~' character is used for it. Most importantly this allows migrating mailboxes from other IMAP servers that contain mailbox names that Dovecot's imapc backend couldn't otherwise access. diffstat: src/lib-storage/index/imapc/imapc-list.c | 1 + src/lib-storage/index/imapc/imapc-storage.h | 1 + src/lib-storage/mailbox-list.c | 121 +++++++++++++++++++++++++-- src/lib-storage/mailbox-list.h | 5 + 4 files changed, 116 insertions(+), 12 deletions(-) diffs (200 lines): diff -r a44be96f55c1 -r 22e7322e8601 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 03:47:41 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Fri Oct 12 05:45:42 2012 +0300 @@ -216,6 +216,7 @@ list_set.layout = MAILBOX_LIST_NAME_MAILDIRPLUSPLUS; list_set.root_dir = dir; list_set.escape_char = IMAPC_LIST_ESCAPE_CHAR; + list_set.broken_char = IMAPC_LIST_BROKEN_CHAR; list_set.mailbox_dir_name = ""; list_set.maildir_name = ""; diff -r a44be96f55c1 -r 22e7322e8601 src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Fri Oct 12 03:47:41 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.h Fri Oct 12 05:45:42 2012 +0300 @@ -7,6 +7,7 @@ #define IMAPC_STORAGE_NAME "imapc" #define IMAPC_INDEX_PREFIX "dovecot.index" #define IMAPC_LIST_ESCAPE_CHAR '%' +#define IMAPC_LIST_BROKEN_CHAR '~' struct imap_arg; struct imapc_untagged_reply; diff -r a44be96f55c1 -r 22e7322e8601 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Fri Oct 12 03:47:41 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Fri Oct 12 05:45:42 2012 +0300 @@ -432,6 +432,53 @@ return str_c(escaped_name); } +static int +mailbox_list_unescape_broken_chars(struct mailbox_list *list, char *name) +{ + char *src, *dest; + unsigned char chr; + + if ((src = strchr(name, list->set.broken_char)) == NULL) + return 0; + dest = src; + + while (*src != '\0') { + if (*src == list->set.broken_char) { + if (src[1] >= '0' && src[1] <= '9') + chr = (src[1]-'0') * 0x10; + else if (src[1] >= 'a' && src[1] <= 'f') + chr = (src[1]-'a' + 10) * 0x10; + else + return -1; + + if (src[2] >= '0' && src[2] <= '9') + chr += src[2]-'0'; + else if (src[2] >= 'a' && src[2] <= 'f') + chr += src[2]-'a' + 10; + else + return -1; + *dest++ = chr; + src += 3; + } else { + *dest++ = *src++; + } + } + *dest++ = '\0'; + return 0; +} + +static char *mailbox_list_convert_sep(const char *storage_name, char src, char dest) +{ + char *ret, *p; + + ret = p_strdup(unsafe_data_stack_pool, storage_name); + for (p = ret; *p != '\0'; p++) { + if (*p == src) + *p = dest; + } + return ret; +} + const char *mailbox_list_default_get_storage_name(struct mailbox_list *list, const char *vname) { @@ -439,7 +486,7 @@ unsigned int prefix_len = strlen(ns->prefix); const char *storage_name = vname; string_t *str; - char list_sep, ns_sep, *ret, *p; + char list_sep, ns_sep, *ret; if (strcasecmp(storage_name, "INBOX") == 0 && (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) @@ -480,19 +527,29 @@ storage_name = "INBOX"; } - if (list_sep == ns_sep) + if (list_sep != ns_sep) { + if (ns->type == MAIL_NAMESPACE_TYPE_SHARED && + (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { + /* shared namespace root. the backend storage's + hierarchy separator isn't known yet, so do + nothing. */ + return storage_name; + } + + ret = mailbox_list_convert_sep(storage_name, ns_sep, list_sep); + } else if (list->set.broken_char == '\0' || + strchr(storage_name, list->set.broken_char) == NULL) { + /* no need to convert broken chars */ return storage_name; - if (ns->type == MAIL_NAMESPACE_TYPE_SHARED && - (ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { - /* shared namespace root. the backend storage's hierarchy - separator isn't known yet, so do nothing. */ - return storage_name; + } else { + ret = p_strdup(unsafe_data_stack_pool, storage_name); } - ret = p_strdup(unsafe_data_stack_pool, storage_name); - for (p = ret; *p != '\0'; p++) { - if (*p == ns_sep) - *p = list_sep; + if (list->set.broken_char != '\0') { + if (mailbox_list_unescape_broken_chars(list, ret) < 0) { + ret = mailbox_list_convert_sep(storage_name, + ns_sep, list_sep); + } } return ret; } @@ -539,6 +596,40 @@ return str_c(dest); } +static void +mailbox_list_escape_broken_chars(struct mailbox_list *list, string_t *str) +{ + unsigned int i; + char buf[3]; + + if (strchr(str_c(str), list->set.broken_char) == NULL) + return; + + for (i = 0; i < str_len(str); i++) { + if (str_c(str)[i] == list->set.broken_char) { + i_snprintf(buf, sizeof(buf), "%02x", + list->set.broken_char); + str_insert(str, i+1, buf); + i += 2; + } + } +} + +static void +mailbox_list_escape_broken_name(struct mailbox_list *list, + const char *vname, string_t *str) +{ + str_truncate(str, 0); + for (; *vname != '\0'; vname++) { + if (*vname == '&' || (unsigned char)*vname >= 0x80) { + str_printfa(str, "%c%02x", list->set.broken_char, + *vname); + } else { + str_append_c(str, *vname); + } + } +} + const char *mailbox_list_default_get_vname(struct mailbox_list *list, const char *storage_name) { @@ -573,8 +664,14 @@ } else if (!list->set.utf8) { /* mUTF-7 -> UTF-8 conversion */ string_t *str = t_str_new(strlen(vname)); - if (imap_utf7_to_utf8(vname, str) == 0) + if (imap_utf7_to_utf8(vname, str) == 0) { + if (list->set.broken_char != '\0') + mailbox_list_escape_broken_chars(list, str); vname = str_c(str); + } else if (list->set.broken_char != '\0') { + mailbox_list_escape_broken_name(list, vname, str); + vname = str_c(str); + } } prefix_len = strlen(list->ns->prefix); diff -r a44be96f55c1 -r 22e7322e8601 src/lib-storage/mailbox-list.h --- a/src/lib-storage/mailbox-list.h Fri Oct 12 03:47:41 2012 +0300 +++ b/src/lib-storage/mailbox-list.h Fri Oct 12 05:45:42 2012 +0300 @@ -103,6 +103,11 @@ /* Encode "bad" characters in mailbox names as */ char escape_char; + /* If mailbox name can't be changed reversibly to UTF-8 and back, + encode the problematic parts using in the + user-visible UTF-8 name. The broken_char itself also has to be + encoded the same way. */ + char broken_char; /* Use UTF-8 mailbox names on filesystem instead of mUTF-7 */ bool utf8; /* Don't check/create the alt-dir symlink. */ From dovecot at dovecot.org Fri Oct 12 07:58:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 07:58:29 +0300 Subject: dovecot-2.2: maildir: If INDEXPVT is set, assume private \Seen f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/16a626119f7b changeset: 15211:16a626119f7b user: Timo Sirainen date: Fri Oct 12 07:58:17 2012 +0300 description: maildir: If INDEXPVT is set, assume private \Seen flags even if dovecot-shared doesn't exist. diffstat: src/lib-storage/index/maildir/maildir-storage.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (21 lines): diff -r 22e7322e8601 -r 16a626119f7b src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Fri Oct 12 05:45:42 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Fri Oct 12 07:58:17 2012 +0300 @@ -614,9 +614,14 @@ mbox->private_flags_mask_set = TRUE; path = mailbox_list_get_root_forced(box->list, MAILBOX_LIST_PATH_TYPE_MAILBOX); - if (!mailbox_list_get_root_path(box->list, MAILBOX_LIST_PATH_TYPE_INDEX, - &path2) || - strcmp(path, path2) == 0) { + if (box->list->set.index_pvt_dir != NULL) { + /* private index directory is set. we'll definitely have + private flags. */ + mbox->_private_flags_mask = MAIL_SEEN; + } else if (!mailbox_list_get_root_path(box->list, + MAILBOX_LIST_PATH_TYPE_INDEX, + &path2) || + strcmp(path, path2) == 0) { /* no separate index directory. we can't have private flags, so don't even bother checking if dovecot-shared exists */ } else { From dovecot at dovecot.org Fri Oct 12 08:31:32 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 08:31:32 +0300 Subject: dovecot-2.1: lib-index: Log a warning if locking transaction log... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e44579c5b52b changeset: 14760:e44579c5b52b user: Timo Sirainen date: Fri Oct 12 08:31:15 2012 +0300 description: lib-index: Log a warning if locking transaction log takes longer than 30 secs. diffstat: src/lib-index/mail-transaction-log-private.h | 1 + src/lib-index/mail-transaction-log.c | 7 +++++++ 2 files changed, 8 insertions(+), 0 deletions(-) diffs (49 lines): diff -r 9542732069ff -r e44579c5b52b src/lib-index/mail-transaction-log-private.h --- a/src/lib-index/mail-transaction-log-private.h Fri Oct 12 00:30:23 2012 +0300 +++ b/src/lib-index/mail-transaction-log-private.h Fri Oct 12 08:31:15 2012 +0300 @@ -10,6 +10,7 @@ mails. */ #define MAIL_TRANSCATION_LOG_LOCK_TIMEOUT (3*60) #define MAIL_TRANSCATION_LOG_LOCK_CHANGE_TIMEOUT (3*60) +#define MAIL_TRANSACTION_LOG_LOCK_WARN_SECS 30 /* Rotate when log is older than ROTATE_TIME and larger than MIN_SIZE */ #define MAIL_TRANSACTION_LOG_ROTATE_MIN_SIZE (1024*32) diff -r 9542732069ff -r e44579c5b52b src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Fri Oct 12 00:30:23 2012 +0300 +++ b/src/lib-index/mail-transaction-log.c Fri Oct 12 08:31:15 2012 +0300 @@ -416,6 +416,7 @@ int mail_transaction_log_lock_head(struct mail_transaction_log *log) { struct mail_transaction_log_file *file; + time_t lock_wait_started, lock_secs = 0; int ret = 0; if (!log->log_2_unlink_checked) { @@ -437,6 +438,7 @@ can lock it and don't see another file, we can be sure no-one is creating a new log at the moment */ + lock_wait_started = time(NULL); for (;;) { file = log->head; if (mail_transaction_log_file_lock(file) < 0) @@ -451,6 +453,7 @@ if (ret == 0 && log->head == file) { /* success */ + lock_secs = file->lock_created - lock_wait_started; break; } @@ -462,6 +465,10 @@ /* try again */ } + if (lock_secs > MAIL_TRANSACTION_LOG_LOCK_WARN_SECS) { + i_warning("Locking transaction log file %s took %ld seconds", + log->head->filepath, (long)lock_secs); + } i_assert(ret < 0 || log->head != NULL); return ret; From dovecot at dovecot.org Fri Oct 12 08:39:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 08:39:45 +0300 Subject: dovecot-2.2: lib-index: dovecot.index file is no longer overwrit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7ac3897f3fd changeset: 15212:b7ac3897f3fd user: Timo Sirainen date: Fri Oct 12 08:39:31 2012 +0300 description: lib-index: dovecot.index file is no longer overwritten, so it doesn't need to be locked. diffstat: src/lib-index/mail-index-lock.c | 99 ---------------------------------- src/lib-index/mail-index-map-read.c | 16 +---- src/lib-index/mail-index-map.c | 6 -- src/lib-index/mail-index-private.h | 9 +-- src/lib-index/mail-index-sync-update.c | 3 - src/lib-index/mail-index.c | 6 -- 6 files changed, 3 insertions(+), 136 deletions(-) diffs (296 lines): diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index-lock.c --- a/src/lib-index/mail-index-lock.c Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index-lock.c Fri Oct 12 08:39:31 2012 +0300 @@ -36,45 +36,6 @@ timeout_secs, lock_r); } -static int mail_index_lock(struct mail_index *index, - unsigned int timeout_secs, unsigned int *lock_id_r) -{ - int ret; - - if (index->lock_type != F_UNLCK) { - /* file is already locked */ - index->shared_lock_count++; - *lock_id_r = index->lock_id_counter; - return 1; - } - - if (index->lock_method == FILE_LOCK_METHOD_DOTLOCK && - !MAIL_INDEX_IS_IN_MEMORY(index)) { - /* FIXME: exclusive locking will rewrite the index file every - time. shouldn't really be needed.. reading doesn't require - locks then, though */ - index->shared_lock_count++; - index->lock_type = F_RDLCK; - *lock_id_r = index->lock_id_counter; - return 1; - } - - i_assert(index->lock_type == F_UNLCK); - ret = mail_index_lock_fd(index, index->filepath, index->fd, - F_RDLCK, timeout_secs, - &index->file_lock); - if (ret <= 0) - return ret; - - if (index->lock_type == F_UNLCK) - index->lock_id_counter += 2; - index->lock_type = F_RDLCK; - - index->shared_lock_count++; - *lock_id_r = index->lock_id_counter; - return 1; -} - void mail_index_flush_read_cache(struct mail_index *index, const char *path, int fd, bool locked) { @@ -91,63 +52,3 @@ nfs_flush_read_cache_unlocked(path, fd); } } - -int mail_index_lock_shared(struct mail_index *index, unsigned int *lock_id_r) -{ - unsigned int timeout_secs; - int ret; - - timeout_secs = I_MIN(MAIL_INDEX_SHARED_LOCK_TIMEOUT, - index->max_lock_timeout_secs); - ret = mail_index_lock(index, timeout_secs, lock_id_r); - if (ret > 0) { - mail_index_flush_read_cache(index, index->filepath, - index->fd, TRUE); - return 0; - } - if (ret < 0) - return -1; - - mail_index_set_error(index, - "Timeout (%us) while waiting for shared lock for index file %s", - timeout_secs, index->filepath); - index->index_lock_timeout = TRUE; - return -1; -} - -void mail_index_unlock(struct mail_index *index, unsigned int *_lock_id) -{ - unsigned int lock_id = *_lock_id; - - *_lock_id = 0; - - /* shared lock */ - if (!mail_index_is_locked(index, lock_id)) { - /* unlocking some older generation of the index file. - we've already closed the file so just ignore this. */ - return; - } - - i_assert(index->shared_lock_count > 0); - index->shared_lock_count--; - - if (index->shared_lock_count == 0) { - index->lock_id_counter += 2; - index->lock_type = F_UNLCK; - if (index->lock_method != FILE_LOCK_METHOD_DOTLOCK) { - if (!MAIL_INDEX_IS_IN_MEMORY(index)) - file_unlock(&index->file_lock); - } - i_assert(index->file_lock == NULL); - } -} - -bool mail_index_is_locked(struct mail_index *index, unsigned int lock_id) -{ - if ((index->lock_id_counter ^ lock_id) <= 1 && lock_id != 0) { - i_assert(index->lock_type != F_UNLCK); - return TRUE; - } - - return FALSE; -} diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index-map-read.c --- a/src/lib-index/mail-index-map-read.c Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index-map-read.c Fri Oct 12 08:39:31 2012 +0300 @@ -235,8 +235,7 @@ return 1; } -static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size, - unsigned int *lock_id) +static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size) { struct mail_index *index = map->index; mail_index_sync_lost_handler_t *const *handlerp; @@ -264,7 +263,6 @@ /* ESTALE - reopen index file */ mail_index_close_file(index); - *lock_id = 0; ret = mail_index_try_open_only(index); if (ret <= 0) { @@ -275,8 +273,6 @@ } return -1; } - if (mail_index_lock_shared(index, lock_id) < 0) - return -1; if (fstat(index->fd, &st) == 0) file_size = st.st_size; @@ -297,7 +293,6 @@ { struct mail_index_map *old_map, *new_map; struct stat st; - unsigned int lock_id; uoff_t file_size; bool use_mmap, unusable = FALSE; int ret, try; @@ -312,10 +307,6 @@ return 1; } - /* the index file is still open, lock it */ - if (mail_index_lock_shared(index, &lock_id) < 0) - return -1; - if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0) nfs_flush_attr_cache_fd_locked(index->filepath, index->fd); @@ -324,7 +315,6 @@ else { if (!ESTALE_FSTAT(errno)) { mail_index_set_syscall_error(index, "fstat()"); - mail_index_unlock(index, &lock_id); return -1; } file_size = (uoff_t)-1; @@ -337,11 +327,9 @@ new_map = mail_index_map_alloc(index); if (use_mmap) { - new_map->rec_map->lock_id = lock_id; ret = mail_index_mmap(new_map, file_size); } else { - ret = mail_index_read_map(new_map, file_size, &lock_id); - mail_index_unlock(index, &lock_id); + ret = mail_index_read_map(new_map, file_size); } if (ret == 0) { /* the index files are unusable */ diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index-map.c --- a/src/lib-index/mail-index-map.c Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index-map.c Fri Oct 12 08:39:31 2012 +0300 @@ -266,9 +266,6 @@ static void mail_index_record_map_free(struct mail_index_map *map, struct mail_index_record_map *rec_map) { - if (rec_map->lock_id != 0) - mail_index_unlock(map->index, &rec_map->lock_id); - if (rec_map->buffer != NULL) { i_assert(rec_map->mmap_base == NULL); buffer_free(&rec_map->buffer); @@ -466,8 +463,6 @@ if (map->rec_map->mmap_base == NULL) return; - i_assert(map->rec_map->lock_id != 0); - if (array_count(&map->rec_map->maps) == 1) new_map = map->rec_map; else { @@ -484,7 +479,6 @@ mail_index_record_map_unlink(map); map->rec_map = new_map; } else { - mail_index_unlock(map->index, &new_map->lock_id); if (munmap(new_map->mmap_base, new_map->mmap_size) < 0) mail_index_set_syscall_error(map->index, "munmap()"); new_map->mmap_base = NULL; diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index-private.h --- a/src/lib-index/mail-index-private.h Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index-private.h Fri Oct 12 08:39:31 2012 +0300 @@ -119,7 +119,6 @@ void *mmap_base; size_t mmap_size, mmap_used_size; - unsigned int lock_id; buffer_t *buffer; @@ -205,7 +204,7 @@ /* syncing will update this if non-NULL */ struct mail_index_transaction_commit_result *sync_commit_result; - int lock_type, shared_lock_count; + int lock_type; unsigned int lock_id_counter; enum file_lock_method lock_method; unsigned int max_lock_timeout_secs; @@ -273,12 +272,6 @@ void mail_index_flush_read_cache(struct mail_index *index, const char *path, int fd, bool locked); -/* Returns 0 = ok, -1 = error. */ -int mail_index_lock_shared(struct mail_index *index, unsigned int *lock_id_r); -void mail_index_unlock(struct mail_index *index, unsigned int *lock_id); -/* Returns TRUE if given lock_id is valid. */ -bool mail_index_is_locked(struct mail_index *index, unsigned int lock_id); - int mail_index_lock_fd(struct mail_index *index, const char *path, int fd, int lock_type, unsigned int timeout_secs, struct file_lock **lock_r); diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index-sync-update.c Fri Oct 12 08:39:31 2012 +0300 @@ -75,9 +75,6 @@ { struct mail_index_map *map = ctx->view->map; - i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) || - map->rec_map->lock_id != 0); - if (map->refcount > 1) { map = mail_index_map_clone(map); mail_index_sync_replace_map(ctx, map); diff -r 16a626119f7b -r b7ac3897f3fd src/lib-index/mail-index.c --- a/src/lib-index/mail-index.c Fri Oct 12 07:58:17 2012 +0300 +++ b/src/lib-index/mail-index.c Fri Oct 12 08:39:31 2012 +0300 @@ -424,7 +424,6 @@ if (MAIL_INDEX_IS_IN_MEMORY(index)) return 0; - i_assert(index->map == NULL || index->map->rec_map->lock_id == 0); ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD); if (ret == 0) { /* it's corrupted - recreate it */ @@ -571,7 +570,6 @@ i_strdup("(in-memory index)") : i_strconcat(index->dir, "/", index->prefix, NULL); - index->shared_lock_count = 0; index->lock_type = F_UNLCK; index->lock_id_counter = 2; @@ -629,7 +627,6 @@ index->lock_id_counter += 2; index->lock_type = F_UNLCK; - index->shared_lock_count = 0; } void mail_index_close(struct mail_index *index) @@ -693,9 +690,6 @@ { struct stat st1, st2; - i_assert(index->shared_lock_count == 0 || - (index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) == 0); - if (MAIL_INDEX_IS_IN_MEMORY(index)) return 0; From dovecot at dovecot.org Fri Oct 12 08:48:35 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 08:48:35 +0300 Subject: dovecot-2.1: lib-storage: Fixed crash when attempting to remove ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/a79e1a0040de changeset: 14761:a79e1a0040de user: Timo Sirainen date: Fri Oct 12 08:48:25 2012 +0300 description: lib-storage: Fixed crash when attempting to remove subscriptions for nonexistent shared users. diffstat: src/lib-storage/list/mailbox-list-subscriptions.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r e44579c5b52b -r a79e1a0040de src/lib-storage/list/mailbox-list-subscriptions.c --- a/src/lib-storage/list/mailbox-list-subscriptions.c Fri Oct 12 08:31:15 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-subscriptions.c Fri Oct 12 08:48:25 2012 +0300 @@ -110,6 +110,8 @@ char sep; int ret; + /* src_list is subscriptions=yes, dest_list is subscriptions=no + (or the same as src_list) */ i_assert((src_list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0); if (dest_list->subscriptions == NULL) { @@ -151,8 +153,8 @@ i_warning("Subscriptions file %s: " "Removing invalid entry: %s", path, name); - (void)subsfile_set_subscribed(dest_list, path, - mailbox_list_get_temp_prefix(dest_list), + (void)subsfile_set_subscribed(src_list, path, + mailbox_list_get_temp_prefix(src_list), name, FALSE); } From pigeonhole at rename-it.nl Fri Oct 12 20:25:16 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 12 Oct 2012 19:25:16 +0200 Subject: dovecot-2.2-pigeonhole: Incorporated Dovecot plugin ABI changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c7db8f43e414 changeset: 1688:c7db8f43e414 user: Stephan Bosch date: Fri Oct 12 00:54:03 2012 +0200 description: Incorporated Dovecot plugin ABI changes. diffstat: src/plugins/lda-sieve/lda-sieve-plugin.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r df0ac1d9be50 -r c7db8f43e414 src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Wed Oct 03 18:00:50 2012 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Fri Oct 12 00:54:03 2012 +0200 @@ -830,7 +830,7 @@ * Plugin interface */ -const char *sieve_plugin_version = DOVECOT_VERSION; +const char *sieve_plugin_version = DOVECOT_ABI_VERSION; const char sieve_plugin_binary_dependency[] = "lda lmtp"; void sieve_plugin_init(void) From pigeonhole at rename-it.nl Fri Oct 12 20:25:17 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 12 Oct 2012 19:25:17 +0200 Subject: dovecot-2.2-pigeonhole: Incorporated Dovecot plugin ABI changes ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/cf079ce8368b changeset: 1689:cf079ce8368b user: Stephan Bosch date: Fri Oct 12 19:25:10 2012 +0200 description: Incorporated Dovecot plugin ABI changes (forgot a few issues). diffstat: src/lib-sieve/sieve-plugins.c | 2 +- src/managesieve-login/managesieve-login-settings-plugin.c | 2 +- src/managesieve/managesieve-settings.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diffs (33 lines): diff -r c7db8f43e414 -r cf079ce8368b src/lib-sieve/sieve-plugins.c --- a/src/lib-sieve/sieve-plugins.c Fri Oct 12 00:54:03 2012 +0200 +++ b/src/lib-sieve/sieve-plugins.c Fri Oct 12 19:25:10 2012 +0200 @@ -77,7 +77,7 @@ path = MODULEDIR"/sieve"; memset(&mod_set, 0, sizeof(mod_set)); - mod_set.version = PIGEONHOLE_VERSION; + mod_set.abi_version = PIGEONHOLE_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = FALSE; diff -r c7db8f43e414 -r cf079ce8368b src/managesieve-login/managesieve-login-settings-plugin.c --- a/src/managesieve-login/managesieve-login-settings-plugin.c Fri Oct 12 00:54:03 2012 +0200 +++ b/src/managesieve-login/managesieve-login-settings-plugin.c Fri Oct 12 19:25:10 2012 +0200 @@ -28,7 +28,7 @@ static void managesieve_login_config_parser_begin(struct config_parser_context *ctx); -const char *managesieve_login_settings_version = DOVECOT_VERSION; +const char *managesieve_login_settings_version = DOVECOT_ABI_VERSION; void managesieve_login_settings_init(struct module *module ATTR_UNUSED) { diff -r c7db8f43e414 -r cf079ce8368b src/managesieve/managesieve-settings.c --- a/src/managesieve/managesieve-settings.c Fri Oct 12 00:54:03 2012 +0200 +++ b/src/managesieve/managesieve-settings.c Fri Oct 12 19:25:10 2012 +0200 @@ -167,4 +167,4 @@ /* */ -const char *managesieve_settings_version = DOVECOT_VERSION; +const char *managesieve_settings_version = DOVECOT_ABI_VERSION; From dovecot at dovecot.org Fri Oct 12 23:05:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 23:05:54 +0300 Subject: dovecot-2.1: configure: Make sure MYSQL_LIBS has -lmysqlclient e... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c8d55ba25f39 changeset: 14762:c8d55ba25f39 user: Timo Sirainen date: Fri Oct 12 23:05:43 2012 +0300 description: configure: Make sure MYSQL_LIBS has -lmysqlclient even if it's not explicitly found. diffstat: configure.in | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a79e1a0040de -r c8d55ba25f39 configure.in --- a/configure.in Fri Oct 12 08:48:25 2012 +0300 +++ b/configure.in Fri Oct 12 23:05:43 2012 +0300 @@ -2175,6 +2175,7 @@ AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, mysql_config, NO) if test $MYSQL_CONFIG = NO; then # based on code from PHP + MYSQL_LIBS="-lmysqlclient -lz -lm" for i in /usr /usr/local /usr/local/mysql; do for j in include include/mysql ""; do if test -r "$i/$j/mysql.h"; then From dovecot at dovecot.org Fri Oct 12 23:19:10 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Oct 2012 23:19:10 +0300 Subject: dovecot-2.2: config: Added lib-master's settings parsers back to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/78749ae0c9c2 changeset: 15213:78749ae0c9c2 user: Timo Sirainen date: Fri Oct 12 23:18:58 2012 +0300 description: config: Added lib-master's settings parsers back to config's knowledge diffstat: src/config/settings-get.pl | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r b7ac3897f3fd -r 78749ae0c9c2 src/config/settings-get.pl --- a/src/config/settings-get.pl Fri Oct 12 08:39:31 2012 +0300 +++ b/src/config/settings-get.pl Fri Oct 12 23:18:58 2012 +0300 @@ -136,6 +136,8 @@ print "};\n"; print "const struct setting_parser_info *all_default_roots[] = {\n"; +print "\t&master_service_setting_parser_info,\n"; +print "\t&master_service_ssl_setting_parser_info,\n"; foreach my $name (keys %parsers) { my $module = $parsers{$name}; next if (!$module); From dovecot at dovecot.org Sat Oct 13 00:24:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 00:24:58 +0300 Subject: dovecot-2.2: lib-ssl-iostream: Make the input buffering behave t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/083bd881b9da changeset: 15214:083bd881b9da user: Timo Sirainen date: Sat Oct 13 00:24:47 2012 +0300 description: lib-ssl-iostream: Make the input buffering behave the same as in file-istream Previously i_stream_read(ssl_input) could have still left some data buffered into the underlying file-istream, which meant that I/O loop didn't detect any new input from the fd and the connection got stuck. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 42 ++++++++++++++++++++--- src/lib-ssl-iostream/iostream-openssl.h | 1 + src/lib-ssl-iostream/istream-openssl.c | 58 +++++++++++++++++++++++++++++--- 3 files changed, 88 insertions(+), 13 deletions(-) diffs (178 lines): diff -r 78749ae0c9c2 -r 083bd881b9da src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Fri Oct 12 23:18:58 2012 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.c Sat Oct 13 00:24:47 2012 +0300 @@ -1,7 +1,7 @@ /* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "istream.h" +#include "istream-private.h" #include "ostream-private.h" #include "iostream-openssl.h" @@ -326,18 +326,39 @@ return bytes_sent; } +static ssize_t +ssl_iostream_read_more(struct ssl_iostream *ssl_io, + const unsigned char **data_r, size_t *size_r) +{ + *data_r = i_stream_get_data(ssl_io->plain_input, size_r); + if (*size_r > 0) + return 0; + + if (!ssl_io->input_handler) { + /* read plain_input only when we came here from input handler. + this makes sure that we don't get stuck with some input + unexpectedly buffered. */ + return 0; + } + + if (i_stream_read_data(ssl_io->plain_input, data_r, size_r, 0) < 0) + return -1; + return 0; +} + static bool ssl_iostream_bio_input(struct ssl_iostream *ssl_io) { const unsigned char *data; size_t bytes, size; + int ret; bool bytes_read = FALSE; - int ret; while ((bytes = BIO_ctrl_get_write_guarantee(ssl_io->bio_ext)) > 0) { /* bytes contains how many bytes we can write to bio_ext */ - if (i_stream_read_data(ssl_io->plain_input, - &data, &size, 0) == -1 && - size == 0 && !bytes_read) { + ssl_io->plain_input->real_stream->try_alloc_limit = bytes; + ret = ssl_iostream_read_more(ssl_io, &data, &size); + ssl_io->plain_input->real_stream->try_alloc_limit = 0; + if (ret == -1 && size == 0 && !bytes_read) { ssl_io->plain_stream_errno = ssl_io->plain_input->stream_errno; ssl_io->closed = TRUE; @@ -358,7 +379,16 @@ } if (bytes == 0 && !bytes_read && ssl_io->want_read) { /* shouldn't happen */ - i_panic("SSL BIO buffer size too small"); + i_error("SSL BIO buffer size too small"); + ssl_io->plain_stream_errno = EINVAL; + ssl_io->closed = TRUE; + return FALSE; + } + if (i_stream_get_data_size(ssl_io->plain_input) > 0) { + i_error("SSL: Too much data in buffered plain input buffer"); + ssl_io->plain_stream_errno = EINVAL; + ssl_io->closed = TRUE; + return FALSE; } if (bytes_read) { if (ssl_io->ostream_flush_waiting_input) { diff -r 78749ae0c9c2 -r 083bd881b9da src/lib-ssl-iostream/iostream-openssl.h --- a/src/lib-ssl-iostream/iostream-openssl.h Fri Oct 12 23:18:58 2012 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.h Sat Oct 13 00:24:47 2012 +0300 @@ -47,6 +47,7 @@ unsigned int cert_received:1; unsigned int cert_broken:1; unsigned int want_read:1; + unsigned int input_handler:1; unsigned int ostream_flush_waiting_input:1; unsigned int closed:1; }; diff -r 78749ae0c9c2 -r 083bd881b9da src/lib-ssl-iostream/istream-openssl.c --- a/src/lib-ssl-iostream/istream-openssl.c Fri Oct 12 23:18:58 2012 +0300 +++ b/src/lib-ssl-iostream/istream-openssl.c Sat Oct 13 00:24:47 2012 +0300 @@ -25,17 +25,27 @@ ssl_iostream_unref(&sstream->ssl_io); } -static ssize_t i_stream_ssl_read(struct istream_private *stream) +static ssize_t i_stream_ssl_read_real(struct istream_private *stream) { struct ssl_istream *sstream = (struct ssl_istream *)stream; + struct ssl_iostream *ssl_io = sstream->ssl_io; + unsigned char buffer[IO_BLOCK_SIZE]; + size_t orig_max_buffer_size = stream->max_buffer_size; size_t size; - ssize_t ret; + ssize_t ret, total_ret; if (sstream->seen_eof) { stream->istream.eof = TRUE; return -1; } - ret = ssl_iostream_more(sstream->ssl_io); + + if (stream->pos >= stream->max_buffer_size) { + i_stream_compress(stream); + if (stream->pos >= stream->max_buffer_size) + return -2; + } + + ret = ssl_iostream_more(ssl_io); if (ret <= 0) { if (ret < 0) { /* handshake failed */ @@ -46,13 +56,16 @@ } if (!i_stream_try_alloc(stream, 1, &size)) - return -2; + i_unreached(); + if (stream->pos + size > stream->max_buffer_size) { + i_assert(stream->max_buffer_size > stream->pos); + size = stream->max_buffer_size - stream->pos; + } - while ((ret = SSL_read(sstream->ssl_io->ssl, + while ((ret = SSL_read(ssl_io->ssl, stream->w_buffer + stream->pos, size)) <= 0) { /* failed to read anything */ - ret = ssl_iostream_handle_error(sstream->ssl_io, ret, - "SSL_read"); + ret = ssl_iostream_handle_error(ssl_io, ret, "SSL_read"); if (ret <= 0) { if (ret < 0) { stream->istream.stream_errno = errno; @@ -64,6 +77,37 @@ /* we did some BIO I/O, try reading again */ } stream->pos += ret; + total_ret = ret; + + /* now make sure that we read everything already buffered in OpenSSL + into the stream (without reading anything more). this makes I/O loop + behave similary for ssl-istream as file-istream. */ + sstream->ssl_io->input_handler = FALSE; + stream->max_buffer_size = (size_t)-1; + while ((ret = SSL_read(ssl_io->ssl, buffer, sizeof(buffer))) > 0) { + if (!i_stream_try_alloc(stream, ret, &size)) + i_unreached(); + i_assert(size >= (size_t)ret); + memcpy(stream->w_buffer + stream->pos, buffer, ret); + stream->pos += ret; + total_ret += ret; + } + stream->max_buffer_size = orig_max_buffer_size; + return total_ret; +} + +static ssize_t i_stream_ssl_read(struct istream_private *stream) +{ + struct ssl_istream *sstream = (struct ssl_istream *)stream; + ssize_t ret; + + sstream->ssl_io->input_handler = TRUE; + if ((ret = i_stream_ssl_read_real(stream)) < 0) + ret = -1; + else { + i_assert(i_stream_get_data_size(sstream->ssl_io->plain_input) == 0); + } + sstream->ssl_io->input_handler = FALSE; return ret; } From dovecot at dovecot.org Sat Oct 13 00:32:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 00:32:47 +0300 Subject: dovecot-2.2: istream: Added internal try_alloc_limit to limit si... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cafdd35ac437 changeset: 15215:cafdd35ac437 user: Timo Sirainen date: Sat Oct 13 00:32:34 2012 +0300 description: istream: Added internal try_alloc_limit to limit size returned by i_stream_try_alloc() This was required by the previous SSL change. diffstat: src/lib/istream-private.h | 2 +- src/lib/istream.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diffs (27 lines): diff -r 083bd881b9da -r cafdd35ac437 src/lib/istream-private.h --- a/src/lib/istream-private.h Sat Oct 13 00:24:47 2012 +0300 +++ b/src/lib/istream-private.h Sat Oct 13 00:32:34 2012 +0300 @@ -29,7 +29,7 @@ unsigned char *w_buffer; /* may be NULL */ size_t buffer_size, max_buffer_size, init_buffer_size; - size_t skip, pos; + size_t skip, pos, try_alloc_limit; struct istream *parent; /* for filter streams */ uoff_t parent_start_offset; diff -r 083bd881b9da -r cafdd35ac437 src/lib/istream.c --- a/src/lib/istream.c Sat Oct 13 00:24:47 2012 +0300 +++ b/src/lib/istream.c Sat Oct 13 00:32:34 2012 +0300 @@ -524,7 +524,10 @@ } *size_r = stream->buffer_size - stream->pos; - return stream->pos != stream->buffer_size; + if (stream->try_alloc_limit > 0 && + *size_r > stream->try_alloc_limit) + *size_r = stream->try_alloc_limit; + return *size_r > 0; } void *i_stream_alloc(struct istream_private *stream, size_t size) From dovecot at dovecot.org Sat Oct 13 00:41:16 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 00:41:16 +0300 Subject: dovecot-2.2: Moved doveadm zlib commands from zlib plugin to dov... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cea1d5504a84 changeset: 15216:cea1d5504a84 user: Timo Sirainen date: Sat Oct 13 00:40:41 2012 +0300 description: Moved doveadm zlib commands from zlib plugin to doveadm directly. Previously it was a plugin, because the istream-zlib existed only in zlib plugin. Now there's a lib-compression that implements it. diffstat: src/doveadm/Makefile.am | 7 +- src/doveadm/doveadm-dump.c | 3 +- src/doveadm/doveadm-dump.h | 1 + src/doveadm/doveadm-zlib.c | 190 +++++++++++++++++++++++++++++++++++++ src/doveadm/doveadm.c | 3 +- src/doveadm/doveadm.h | 1 + src/plugins/zlib/Makefile.am | 12 +-- src/plugins/zlib/doveadm-zlib.c | 205 ---------------------------------------- 8 files changed, 202 insertions(+), 220 deletions(-) diffs (truncated from 514 to 300 lines): diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Sat Oct 13 00:32:34 2012 +0300 +++ b/src/doveadm/Makefile.am Sat Oct 13 00:40:41 2012 +0300 @@ -10,6 +10,7 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-compression \ -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-master \ @@ -32,7 +33,8 @@ ../lib-otp/libotp.a libs = \ - dsync/libdsync.la + dsync/libdsync.la \ + ../lib-compression/libcompression.la doveadm_LDADD = \ $(libs) \ @@ -102,7 +104,8 @@ doveadm-pw.c \ doveadm-sis.c \ doveadm-stats.c \ - doveadm-who.c + doveadm-who.c \ + doveadm-zlib.c doveadm_server_SOURCES = \ $(common) \ diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/doveadm-dump.c --- a/src/doveadm/doveadm-dump.c Sat Oct 13 00:32:34 2012 +0300 +++ b/src/doveadm/doveadm-dump.c Sat Oct 13 00:40:41 2012 +0300 @@ -86,7 +86,8 @@ &doveadm_cmd_dump_index, &doveadm_cmd_dump_log, &doveadm_cmd_dump_mailboxlog, - &doveadm_cmd_dump_thread + &doveadm_cmd_dump_thread, + &doveadm_cmd_dump_zlib }; void print_dump_types(void) diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/doveadm-dump.h --- a/src/doveadm/doveadm-dump.h Sat Oct 13 00:32:34 2012 +0300 +++ b/src/doveadm/doveadm-dump.h Sat Oct 13 00:40:41 2012 +0300 @@ -14,6 +14,7 @@ extern struct doveadm_cmd_dump doveadm_cmd_dump_log; extern struct doveadm_cmd_dump doveadm_cmd_dump_mailboxlog; extern struct doveadm_cmd_dump doveadm_cmd_dump_thread; +extern struct doveadm_cmd_dump doveadm_cmd_dump_zlib; void doveadm_dump_register(const struct doveadm_cmd_dump *dump); diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/doveadm-zlib.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/doveadm-zlib.c Sat Oct 13 00:40:41 2012 +0300 @@ -0,0 +1,190 @@ +/* Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "istream.h" +#include "ostream.h" +#include "istream-zlib.h" +#include "ostream-zlib.h" +#include "module-dir.h" +#include "master-service.h" +#include "doveadm-dump.h" + +#include +#include +#include + +const char *doveadm_zlib_plugin_version = DOVECOT_ABI_VERSION; + +static void cmd_dump_imapzlib(int argc ATTR_UNUSED, char *argv[]) +{ + struct istream *input, *input2; + const unsigned char *data; + size_t size; + const char *line; + int fd; + + fd = open(argv[1], O_RDONLY); + if (fd < 0) + i_fatal("open(%s) failed: %m", argv[1]); + input = i_stream_create_fd(fd, 1024*32, TRUE); + while ((line = i_stream_read_next_line(input)) != NULL) { + /* skip tag */ + printf("%s\r\n", line); + while (*line != ' ' && *line != '\0') line++; + if (*line == '\0') + continue; + line++; + + if (strcmp(line, "OK Begin compression.") == 0 || + strcasecmp(line, "COMPRESS DEFLATE") == 0) + break; + } + + input2 = i_stream_create_deflate(input, TRUE); + i_stream_unref(&input); + + while (i_stream_read_data(input2, &data, &size, 0) != -1) { + fwrite(data, 1, size, stdout); + i_stream_skip(input2, size); + } + i_stream_unref(&input2); + fflush(stdout); +} + +static bool test_dump_imapzlib(const char *path) +{ + const char *p; + char buf[4096]; + int fd, ret; + bool match = FALSE; + + p = strrchr(path, '.'); + if (p == NULL || (strcmp(p, ".in") != 0 && strcmp(p, ".out") != 0)) + return FALSE; + + fd = open(path, O_RDONLY); + if (fd == -1) + return FALSE; + + ret = read(fd, buf, sizeof(buf)-1); + if (ret > 0) { + buf[ret] = '\0'; + (void)str_lcase(buf); + match = strstr(buf, " ok begin compression.") != NULL || + strstr(buf, " compress deflate") != NULL; + } + i_close_fd(&fd); + return match; +} + +struct client { + int fd; + struct io *io_client, *io_server; + struct istream *input; + struct ostream *output; + bool compressed; +}; + +static void client_input(struct client *client) +{ + struct istream *input; + struct ostream *output; + unsigned char buf[1024]; + ssize_t ret; + + ret = read(STDIN_FILENO, buf, sizeof(buf)); + if (ret == 0) { + if (client->compressed) { + master_service_stop(master_service); + return; + } + /* start compression */ + i_info(""); + input = i_stream_create_deflate(client->input, TRUE); + output = o_stream_create_deflate(client->output, 6); + i_stream_unref(&client->input); + o_stream_unref(&client->output); + client->input = input; + client->output = output; + client->compressed = TRUE; + return; + } + if (ret < 0) + i_fatal("read(stdin) failed: %m"); + + o_stream_nsend(client->output, buf, ret); +} + +static void server_input(struct client *client) +{ + const unsigned char *data; + size_t size; + + if (i_stream_read(client->input) == -1) { + if (client->input->stream_errno != 0) { + errno = client->input->stream_errno; + i_fatal("read(server) failed: %m"); + } + + i_info("Server disconnected"); + master_service_stop(master_service); + return; + } + + data = i_stream_get_data(client->input, &size); + if (write(STDOUT_FILENO, data, size) < 0) + i_fatal("write(stdout) failed: %m"); + i_stream_skip(client->input, size); +} + +static void cmd_zlibconnect(int argc ATTR_UNUSED, char *argv[]) +{ + struct client client; + struct ip_addr *ips; + unsigned int ips_count, port = 143; + int fd, ret; + + if (argv[1] == NULL || + (argv[2] != NULL && str_to_uint(argv[2], &port) < 0)) + help(&doveadm_cmd_zlibconnect); + + ret = net_gethostbyname(argv[1], &ips, &ips_count); + if (ret != 0) { + i_fatal("Host %s lookup failed: %s", argv[1], + net_gethosterror(ret)); + } + + if ((fd = net_connect_ip(&ips[0], port, NULL)) == -1) + i_fatal("connect(%s, %u) failed: %m", argv[1], port); + + i_info("Connected to %s port %u. Ctrl-D starts compression", + net_ip2addr(&ips[0]), port); + + memset(&client, 0, sizeof(client)); + client.fd = fd; + client.input = i_stream_create_fd(fd, (size_t)-1, FALSE); + client.output = o_stream_create_fd(fd, 0, FALSE); + o_stream_set_no_error_handling(client.output, TRUE); + client.io_client = io_add(STDIN_FILENO, IO_READ, client_input, &client); + client.io_server = io_add(fd, IO_READ, server_input, &client); + master_service_run(master_service, NULL); + io_remove(&client.io_client); + io_remove(&client.io_server); + i_stream_unref(&client.input); + o_stream_unref(&client.output); + if (close(fd) < 0) + i_fatal("close() failed: %m"); +} + +struct doveadm_cmd_dump doveadm_cmd_dump_zlib = { + "imapzlib", + test_dump_imapzlib, + cmd_dump_imapzlib +}; + +struct doveadm_cmd doveadm_cmd_zlibconnect = { + cmd_zlibconnect, + "zlibconnect", + " []" +}; diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/doveadm.c --- a/src/doveadm/doveadm.c Sat Oct 13 00:32:34 2012 +0300 +++ b/src/doveadm/doveadm.c Sat Oct 13 00:40:41 2012 +0300 @@ -300,7 +300,8 @@ &doveadm_cmd_sis_deduplicate, &doveadm_cmd_sis_find, &doveadm_cmd_stats_dump, - &doveadm_cmd_stats_top + &doveadm_cmd_stats_top, + &doveadm_cmd_zlibconnect }; int main(int argc, char *argv[]) diff -r cafdd35ac437 -r cea1d5504a84 src/doveadm/doveadm.h --- a/src/doveadm/doveadm.h Sat Oct 13 00:32:34 2012 +0300 +++ b/src/doveadm/doveadm.h Sat Oct 13 00:40:41 2012 +0300 @@ -32,6 +32,7 @@ extern struct doveadm_cmd doveadm_cmd_sis_find; extern struct doveadm_cmd doveadm_cmd_stats_dump; extern struct doveadm_cmd doveadm_cmd_stats_top; +extern struct doveadm_cmd doveadm_cmd_zlibconnect; void doveadm_register_cmd(const struct doveadm_cmd *cmd); diff -r cafdd35ac437 -r cea1d5504a84 src/plugins/zlib/Makefile.am --- a/src/plugins/zlib/Makefile.am Sat Oct 13 00:32:34 2012 +0300 +++ b/src/plugins/zlib/Makefile.am Sat Oct 13 00:40:41 2012 +0300 @@ -1,5 +1,3 @@ -doveadm_moduledir = $(moduledir)/doveadm - AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-master \ @@ -8,11 +6,9 @@ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/lib-storage/index \ - -I$(top_srcdir)/src/lib-storage/index/dbox-common \ - -I$(top_srcdir)/src/doveadm + -I$(top_srcdir)/src/lib-storage/index/dbox-common NOPLUGIN_LDFLAGS = -lib10_doveadm_zlib_plugin_la_LDFLAGS = -module -avoid-version lib20_zlib_plugin_la_LDFLAGS = -module -avoid-version module_LTLIBRARIES = \ @@ -26,9 +22,3 @@ noinst_HEADERS = \ zlib-plugin.h - From dovecot at dovecot.org Sat Oct 13 01:09:51 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 01:09:51 +0300 Subject: dovecot-2.2: lib-settings: settings_read*() now returns an error... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/444a9921205a changeset: 15217:444a9921205a user: Timo Sirainen date: Sat Oct 13 01:03:13 2012 +0300 description: lib-settings: settings_read*() now returns an error string instead of logging itself diffstat: src/auth/db-dict.c | 5 +++-- src/auth/db-ldap.c | 6 +++--- src/auth/db-sql.c | 5 +++-- src/lib-dict/dict-sql-settings.c | 5 ++++- src/lib-settings/settings.c | 27 ++++++++------------------- src/lib-settings/settings.h | 11 ++++++----- 6 files changed, 27 insertions(+), 32 deletions(-) diffs (176 lines): diff -r cea1d5504a84 -r 444a9921205a src/auth/db-dict.c --- a/src/auth/db-dict.c Sat Oct 13 00:40:41 2012 +0300 +++ b/src/auth/db-dict.c Sat Oct 13 01:03:13 2012 +0300 @@ -62,6 +62,7 @@ struct dict_connection *db_dict_init(const char *config_path) { struct dict_connection *conn; + const char *error; pool_t pool; conn = dict_conn_find(config_path); @@ -81,8 +82,8 @@ conn->config_path = p_strdup(pool, config_path); conn->set = default_dict_settings; - if (!settings_read_nosection(config_path, parse_setting, conn)) - exit(FATAL_DEFAULT); + if (!settings_read_nosection(config_path, parse_setting, conn, &error)) + i_fatal("dict %s: %s", config_path, error); if (conn->set.uri == NULL) i_fatal("dict %s: Empty uri setting", config_path); diff -r cea1d5504a84 -r 444a9921205a src/auth/db-ldap.c --- a/src/auth/db-ldap.c Sat Oct 13 00:40:41 2012 +0300 +++ b/src/auth/db-ldap.c Sat Oct 13 01:03:13 2012 +0300 @@ -1376,7 +1376,7 @@ struct ldap_connection *db_ldap_init(const char *config_path, bool userdb) { struct ldap_connection *conn; - const char *str; + const char *str, *error; pool_t pool; /* see if it already exists */ @@ -1402,8 +1402,8 @@ conn->fd = -1; conn->config_path = p_strdup(pool, config_path); conn->set = default_ldap_settings; - if (!settings_read_nosection(config_path, parse_setting, conn)) - exit(FATAL_DEFAULT); + if (!settings_read_nosection(config_path, parse_setting, conn, &error)) + i_fatal("ldap %s: %s", config_path, error); if (conn->set.base == NULL) i_fatal("LDAP: No base given"); diff -r cea1d5504a84 -r 444a9921205a src/auth/db-sql.c --- a/src/auth/db-sql.c Sat Oct 13 00:40:41 2012 +0300 +++ b/src/auth/db-sql.c Sat Oct 13 01:03:13 2012 +0300 @@ -64,6 +64,7 @@ struct sql_connection *db_sql_init(const char *config_path, bool userdb) { struct sql_connection *conn; + const char *error; pool_t pool; conn = sql_conn_find(config_path); @@ -86,8 +87,8 @@ conn->config_path = p_strdup(pool, config_path); conn->set = default_sql_settings; - if (!settings_read_nosection(config_path, parse_setting, conn)) - exit(FATAL_DEFAULT); + if (!settings_read_nosection(config_path, parse_setting, conn, &error)) + i_fatal("sql %s: %s", config_path, error); if (conn->set.password_query == default_sql_settings.password_query) conn->default_password_query = TRUE; diff -r cea1d5504a84 -r 444a9921205a src/lib-dict/dict-sql-settings.c --- a/src/lib-dict/dict-sql-settings.c Sat Oct 13 00:40:41 2012 +0300 +++ b/src/lib-dict/dict-sql-settings.c Sat Oct 13 01:03:13 2012 +0300 @@ -213,6 +213,7 @@ struct dict_sql_settings *dict_sql_settings_read(pool_t pool, const char *path) { struct setting_parser_ctx ctx; + const char *error; memset(&ctx, 0, sizeof(ctx)); ctx.pool = pool; @@ -220,8 +221,10 @@ t_array_init(&ctx.cur_fields, 16); p_array_init(&ctx.set->maps, pool, 8); - if (!settings_read(path, NULL, parse_setting, parse_section, &ctx)) + if (!settings_read(path, NULL, parse_setting, parse_section, &ctx, &error)) { + i_error("Error in configuration file %s: %s", path, error); return NULL; + } if (ctx.set->connect == NULL) { i_error("Error in configuration file %s: " diff -r cea1d5504a84 -r 444a9921205a src/lib-settings/settings.c --- a/src/lib-settings/settings.c Sat Oct 13 00:40:41 2012 +0300 +++ b/src/lib-settings/settings.c Sat Oct 13 01:03:13 2012 +0300 @@ -167,10 +167,10 @@ #define IS_WHITE(c) ((c) == ' ' || (c) == '\t') -static bool -settings_read_real(const char *path, const char *section, - settings_callback_t *callback, - settings_section_callback_t *sect_callback, void *context) +bool settings_read_i(const char *path, const char *section, + settings_callback_t *callback, + settings_section_callback_t *sect_callback, void *context, + const char **error_r) { /* pretty horrible code, but v2.0 will have this rewritten anyway.. */ struct input_stack root, *input; @@ -182,7 +182,8 @@ fd = open(path, O_RDONLY); if (fd < 0) { - i_error("Can't open configuration file %s: %m", path); + *error_r = t_strdup_printf( + "Can't open configuration file %s: %m", path); return FALSE; } @@ -371,7 +372,8 @@ } if (errormsg != NULL) { - i_error("Error in configuration file %s line %d: %s", + *error_r = t_strdup_printf( + "Error in configuration file %s line %d: %s", input->path, input->linenum, errormsg); break; } @@ -385,16 +387,3 @@ return errormsg == NULL; } - -bool settings_read_i(const char *path, const char *section, - settings_callback_t *callback, - settings_section_callback_t *sect_callback, void *context) -{ - bool ret; - - T_BEGIN { - ret = settings_read_real(path, section, callback, - sect_callback, context); - } T_END; - return ret; -} diff -r cea1d5504a84 -r 444a9921205a src/lib-settings/settings.h --- a/src/lib-settings/settings.h Sat Oct 13 00:40:41 2012 +0300 +++ b/src/lib-settings/settings.h Sat Oct 13 01:03:13 2012 +0300 @@ -42,9 +42,10 @@ bool settings_read_i(const char *path, const char *section, settings_callback_t *callback, - settings_section_callback_t *sect_callback, void *context) + settings_section_callback_t *sect_callback, void *context, + const char **error_r) ATTR_NULL(2, 4, 5); -#define settings_read(path, section, callback, sect_callback, context) \ +#define settings_read(path, section, callback, sect_callback, context, error_r) \ settings_read_i(path + \ CALLBACK_TYPECHECK(callback, const char *(*)( \ const char *, const char *, typeof(context))) + \ @@ -52,11 +53,11 @@ const char *, const char *, typeof(context), \ const char **)), \ section, (settings_callback_t *)callback, \ - (settings_section_callback_t *)sect_callback, context) -#define settings_read_nosection(path, callback, context) \ + (settings_section_callback_t *)sect_callback, context, error_r) +#define settings_read_nosection(path, callback, context, error_r) \ settings_read_i(path + \ CALLBACK_TYPECHECK(callback, const char *(*)( \ const char *, const char *, typeof(context))), \ - NULL, (settings_callback_t *)callback, NULL, context) + NULL, (settings_callback_t *)callback, NULL, context, error_r) #endif From dovecot at dovecot.org Sat Oct 13 01:09:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 01:09:52 +0300 Subject: dovecot-2.2: lib-dict: dict_init() now returns error string inst... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22c22d704422 changeset: 15218:22c22d704422 user: Timo Sirainen date: Sat Oct 13 01:09:37 2012 +0300 description: lib-dict: dict_init() now returns error string instead of logging it diffstat: src/auth/db-dict.c | 4 ++-- src/dict/dict-connection.c | 7 ++++--- src/lib-dict/dict-client.c | 7 ++++--- src/lib-dict/dict-file.c | 5 +++-- src/lib-dict/dict-memcached-ascii.c | 14 +++++++++----- src/lib-dict/dict-memcached.c | 15 ++++++++++----- src/lib-dict/dict-private.h | 2 +- src/lib-dict/dict-redis.c | 15 ++++++++++----- src/lib-dict/dict-sql-settings.c | 13 ++++++------- src/lib-dict/dict-sql-settings.h | 3 ++- src/lib-dict/dict-sql.c | 4 ++-- src/lib-dict/dict.c | 28 ++++++++++++++++------------ src/lib-dict/dict.h | 3 ++- src/lib-imap-urlauth/imap-urlauth-backend.c | 5 ++++- src/lib-storage/list/mailbox-list-fs-iter.c | 2 +- src/plugins/acl/acl-lookup-dict.c | 6 +++--- src/plugins/expire/doveadm-expire.c | 7 ++++--- src/plugins/expire/expire-plugin.c | 10 ++++++---- src/plugins/quota/quota-dict.c | 7 +++++-- 19 files changed, 94 insertions(+), 63 deletions(-) diffs (truncated from 504 to 300 lines): diff -r 444a9921205a -r 22c22d704422 src/auth/db-dict.c --- a/src/auth/db-dict.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/auth/db-dict.c Sat Oct 13 01:09:37 2012 +0300 @@ -92,8 +92,8 @@ config_path, conn->set.value_format); } if (dict_init(conn->set.uri, DICT_DATA_TYPE_STRING, "", - global_auth_settings->base_dir, &conn->dict) < 0) - i_fatal("dict %s: Failed to init dict", config_path); + global_auth_settings->base_dir, &conn->dict, &error) < 0) + i_fatal("dict %s: Failed to init dict: %s", config_path, error); conn->next = connections; connections = conn; diff -r 444a9921205a -r 22c22d704422 src/dict/dict-connection.c --- a/src/dict/dict-connection.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/dict/dict-connection.c Sat Oct 13 01:09:37 2012 +0300 @@ -68,7 +68,7 @@ { const char *const *strlist; unsigned int i, count; - const char *uri; + const char *uri, *error; strlist = array_get(&dict_settings->dicts, &count); for (i = 0; i < count; i += 2) { @@ -84,9 +84,10 @@ uri = strlist[i+1]; if (dict_init(uri, conn->value_type, conn->username, - dict_settings->base_dir, &conn->dict) < 0) { + dict_settings->base_dir, &conn->dict, &error) < 0) { /* dictionary initialization failed */ - i_error("Failed to initialize dictionary '%s'", conn->name); + i_error("Failed to initialize dictionary '%s': %s", + conn->name, error); return -1; } return 0; diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-client.c Sat Oct 13 01:09:37 2012 +0300 @@ -466,7 +466,8 @@ static int client_dict_init(struct dict *driver, const char *uri, enum dict_data_type value_type, const char *username, - const char *base_dir, struct dict **dict_r) + const char *base_dir, struct dict **dict_r, + const char **error_r) { struct client_dict *dict; const char *dest_uri; @@ -475,7 +476,7 @@ /* uri = [] ":" */ dest_uri = strchr(uri, ':'); if (dest_uri == NULL) { - i_error("dict-client: Invalid URI: %s", uri); + *error_r = t_strdup_printf("Invalid URI: %s", uri); return -1; } @@ -497,7 +498,7 @@ } dict->uri = p_strdup(pool, dest_uri + 1); *dict_r = &dict->dict; - return -1; + return 0; } static void client_dict_deinit(struct dict *_dict) diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-file.c Sat Oct 13 01:09:37 2012 +0300 @@ -55,7 +55,8 @@ file_dict_init(struct dict *driver, const char *uri, enum dict_data_type value_type ATTR_UNUSED, const char *username ATTR_UNUSED, - const char *base_dir ATTR_UNUSED, struct dict **dict_r) + const char *base_dir ATTR_UNUSED, struct dict **dict_r, + const char **error_r) { struct file_dict *dict; const char *p; @@ -74,7 +75,7 @@ else if (strcmp(p, "lock=flock") == 0) dict->lock_method = FILE_LOCK_METHOD_FLOCK; else { - i_error("dict file: Invalid parameter: %s", p+1); + *error_r = t_strdup_printf("Invalid parameter: %s", p+1); i_free(dict->path); i_free(dict); return -1; diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-memcached-ascii.c --- a/src/lib-dict/dict-memcached-ascii.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-memcached-ascii.c Sat Oct 13 01:09:37 2012 +0300 @@ -335,7 +335,7 @@ enum dict_data_type value_type ATTR_UNUSED, const char *username, const char *base_dir ATTR_UNUSED, - struct dict **dict_r) + struct dict **dict_r, const char **error_r) { struct memcached_ascii_dict *dict; const char *const *args; @@ -359,12 +359,14 @@ for (; *args != NULL; args++) { if (strncmp(*args, "host=", 5) == 0) { if (net_addr2ip(*args+5, &dict->ip) < 0) { - i_error("Invalid IP: %s", *args+5); + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "port=", 5) == 0) { if (str_to_uint(*args+5, &dict->port) < 0) { - i_error("Invalid port: %s", *args+5); + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "prefix=", 7) == 0) { @@ -372,11 +374,13 @@ dict->key_prefix = i_strdup(*args + 7); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { - i_error("Invalid timeout_msecs: %s", *args+14); + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); ret = -1; } } else { - i_error("Unknown parameter: %s", *args); + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); ret = -1; } } diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-memcached.c --- a/src/lib-dict/dict-memcached.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-memcached.c Sat Oct 13 01:09:37 2012 +0300 @@ -171,7 +171,8 @@ memcached_dict_init(struct dict *driver, const char *uri, enum dict_data_type value_type ATTR_UNUSED, const char *username ATTR_UNUSED, - const char *base_dir ATTR_UNUSED, struct dict **dict_r) + const char *base_dir ATTR_UNUSED, struct dict **dict_r, + const char **error_r) { struct memcached_dict *dict; const char *const *args; @@ -194,12 +195,14 @@ for (; *args != NULL; args++) { if (strncmp(*args, "host=", 5) == 0) { if (net_addr2ip(*args+5, &dict->ip) < 0) { - i_error("Invalid IP: %s", *args+5); + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "port=", 5) == 0) { if (str_to_uint(*args+5, &dict->port) < 0) { - i_error("Invalid port: %s", *args+5); + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "prefix=", 7) == 0) { @@ -207,11 +210,13 @@ dict->key_prefix = i_strdup(*args + 7); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { - i_error("Invalid timeout_msecs: %s", *args+14); + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); ret = -1; } } else { - i_error("Unknown parameter: %s", *args); + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); ret = -1; } } diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-private.h --- a/src/lib-dict/dict-private.h Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-private.h Sat Oct 13 01:09:37 2012 +0300 @@ -7,7 +7,7 @@ int (*init)(struct dict *dict_driver, const char *uri, enum dict_data_type value_type, const char *username, const char *base_dir, - struct dict **dict_r); + struct dict **dict_r, const char **error_r); void (*deinit)(struct dict *dict); int (*wait)(struct dict *dict); diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-redis.c Sat Oct 13 01:09:37 2012 +0300 @@ -302,7 +302,8 @@ redis_dict_init(struct dict *driver, const char *uri, enum dict_data_type value_type ATTR_UNUSED, const char *username, - const char *base_dir ATTR_UNUSED, struct dict **dict_r) + const char *base_dir ATTR_UNUSED, struct dict **dict_r, + const char **error_r) { struct redis_dict *dict; const char *const *args; @@ -325,12 +326,14 @@ for (; *args != NULL; args++) { if (strncmp(*args, "host=", 5) == 0) { if (net_addr2ip(*args+5, &dict->ip) < 0) { - i_error("Invalid IP: %s", *args+5); + *error_r = t_strdup_printf("Invalid IP: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "port=", 5) == 0) { if (str_to_uint(*args+5, &dict->port) < 0) { - i_error("Invalid port: %s", *args+5); + *error_r = t_strdup_printf("Invalid port: %s", + *args+5); ret = -1; } } else if (strncmp(*args, "prefix=", 7) == 0) { @@ -338,11 +341,13 @@ dict->key_prefix = i_strdup(*args + 7); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { - i_error("Invalid timeout_msecs: %s", *args+14); + *error_r = t_strdup_printf( + "Invalid timeout_msecs: %s", *args+14); ret = -1; } } else { - i_error("Unknown parameter: %s", *args); + *error_r = t_strdup_printf("Unknown parameter: %s", + *args); ret = -1; } } diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-sql-settings.c --- a/src/lib-dict/dict-sql-settings.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-sql-settings.c Sat Oct 13 01:09:37 2012 +0300 @@ -210,10 +210,10 @@ return FALSE; } -struct dict_sql_settings *dict_sql_settings_read(pool_t pool, const char *path) +struct dict_sql_settings * +dict_sql_settings_read(pool_t pool, const char *path, const char **error_r) { struct setting_parser_ctx ctx; - const char *error; memset(&ctx, 0, sizeof(ctx)); ctx.pool = pool; @@ -221,14 +221,13 @@ t_array_init(&ctx.cur_fields, 16); p_array_init(&ctx.set->maps, pool, 8); - if (!settings_read(path, NULL, parse_setting, parse_section, &ctx, &error)) { - i_error("Error in configuration file %s: %s", path, error); + if (!settings_read(path, NULL, parse_setting, parse_section, + &ctx, error_r)) return NULL; - } if (ctx.set->connect == NULL) { - i_error("Error in configuration file %s: " - "Missing connect setting", path); + *error_r = t_strdup_printf("Error in configuration file %s: " + "Missing connect setting", path); return NULL; } diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-sql-settings.h --- a/src/lib-dict/dict-sql-settings.h Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-sql-settings.h Sat Oct 13 01:09:37 2012 +0300 @@ -19,6 +19,7 @@ ARRAY(struct dict_sql_map) maps; }; -struct dict_sql_settings *dict_sql_settings_read(pool_t pool, const char *path); +struct dict_sql_settings * +dict_sql_settings_read(pool_t pool, const char *path, const char **error_r); #endif diff -r 444a9921205a -r 22c22d704422 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Sat Oct 13 01:03:13 2012 +0300 +++ b/src/lib-dict/dict-sql.c Sat Oct 13 01:09:37 2012 +0300 @@ -75,7 +75,7 @@ sql_dict_init(struct dict *driver, const char *uri, enum dict_data_type value_type ATTR_UNUSED, const char *username, const char *base_dir ATTR_UNUSED, - struct dict **dict_r) + struct dict **dict_r, const char **error_r) { struct sql_dict *dict; pool_t pool; From dovecot at dovecot.org Sat Oct 13 01:36:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 01:36:31 +0300 Subject: dovecot-2.2: imap urlauth: Don't autofill imap_urlauth_host. Use... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8d8880296645 changeset: 15219:8d8880296645 user: Timo Sirainen date: Sat Oct 13 01:36:13 2012 +0300 description: imap urlauth: Don't autofill imap_urlauth_host. Use "*" value to mean "any host". diffstat: src/imap-urlauth/imap-urlauth-worker.c | 7 +++++-- src/imap/imap-client.c | 2 +- src/lib-imap-urlauth/imap-urlauth.c | 13 ++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diffs (72 lines): diff -r 22c22d704422 -r 8d8880296645 src/imap-urlauth/imap-urlauth-worker.c --- a/src/imap-urlauth/imap-urlauth-worker.c Sat Oct 13 01:09:37 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker.c Sat Oct 13 01:36:13 2012 +0300 @@ -634,8 +634,11 @@ } /* initialize urlauth context */ - if (set->imap_urlauth_dict == NULL || *set->imap_urlauth_dict == '\0') { - i_error("imap_urlauth_dict setting is not configured for user %s", + if (*set->imap_urlauth_host == '\0' || + *set->imap_urlauth_dict == '\0') { + i_error("%s setting is not configured for user %s", + *set->imap_urlauth_host == '\0' ? + "imap_urlauth_host" : "imap_urlauth_dict", mail_user->username); client_send_line(client, "NO"); client_abort(client, "Session aborted: URLAUTH not configured"); diff -r 22c22d704422 -r 8d8880296645 src/imap/imap-client.c --- a/src/imap/imap-client.c Sat Oct 13 01:09:37 2012 +0300 +++ b/src/imap/imap-client.c Sat Oct 13 01:36:13 2012 +0300 @@ -129,7 +129,7 @@ str_append(client->capability_string, " NOTIFY"); } - if (set->imap_urlauth_dict != NULL && *set->imap_urlauth_dict != '\0') { + if (*set->imap_urlauth_host != '\0' && *set->imap_urlauth_dict != '\0') { if (client_init_urlauth(client) == 0 && !explicit_capability) { /* Enable URLAUTH capability only when dict is diff -r 22c22d704422 -r 8d8880296645 src/lib-imap-urlauth/imap-urlauth.c --- a/src/lib-imap-urlauth/imap-urlauth.c Sat Oct 13 01:09:37 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth.c Sat Oct 13 01:36:13 2012 +0300 @@ -26,6 +26,8 @@ #define IMAP_URLAUTH_NORMAL_TIMEOUT_MSECS 5*1000 #define IMAP_URLAUTH_SPECIAL_TIMEOUT_MSECS 3*60*1000 +#define URL_HOST_ALLOW_ANY "*" + int imap_urlauth_init(struct mail_user *user, const struct imap_urlauth_config *config, struct imap_urlauth_context **ctx_r) @@ -34,16 +36,16 @@ struct imap_urlauth_context *uctx; unsigned int timeout; + i_assert(*config->url_host != '\0'); + i_assert(*config->dict_uri != '\0'); + if (imap_urlauth_backend_create(user, config->dict_uri, &backend) < 0) return -1; uctx = i_new(struct imap_urlauth_context, 1); uctx->user = user; uctx->backend = backend; - if (config->url_host != NULL && *config->url_host != '\0') - uctx->url_host = i_strdup(config->url_host); - else - uctx->url_host = i_strdup(my_hostdomain()); + uctx->url_host = i_strdup(config->url_host); uctx->url_port = config->url_port; if (config->access_anonymous) @@ -210,7 +212,8 @@ { /* validate host */ /* FIXME: allow host ip/ip6 as well? */ - if (strcmp(url->host_name, uctx->url_host) != 0) { + if (strcmp(uctx->url_host, URL_HOST_ALLOW_ANY) != 0 && + strcmp(url->host_name, uctx->url_host) != 0) { *error_r = "Invalid URL: Inappropriate host name"; return FALSE; } From dovecot at dovecot.org Sat Oct 13 01:38:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 01:38:53 +0300 Subject: dovecot-2.2: example-config: Added imap_urlauth_* settings Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dbefcfb9d330 changeset: 15220:dbefcfb9d330 user: Timo Sirainen date: Sat Oct 13 01:38:48 2012 +0300 description: example-config: Added imap_urlauth_* settings diffstat: doc/example-config/conf.d/20-imap.conf | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 8d8880296645 -r dbefcfb9d330 doc/example-config/conf.d/20-imap.conf --- a/doc/example-config/conf.d/20-imap.conf Sat Oct 13 01:36:13 2012 +0300 +++ b/doc/example-config/conf.d/20-imap.conf Sat Oct 13 01:38:48 2012 +0300 @@ -56,3 +56,9 @@ # The list is space-separated. #imap_client_workarounds = } + +# Dictionary containing URLAUTH data. +#imap_urlauth_dict = + +# Host allowed in URLAUTH URLs sent by client. "*" allows all. +#imap_urlauth_host = From dovecot at dovecot.org Sat Oct 13 02:08:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 02:08:22 +0300 Subject: dovecot-2.2: imap-urlauth: Use mailbox GUID instead of mailbox n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/327dc9a8decc changeset: 15221:327dc9a8decc user: Timo Sirainen date: Sat Oct 13 01:57:24 2012 +0300 description: imap-urlauth: Use mailbox GUID instead of mailbox name for dict keys diffstat: src/imap/cmd-resetkey.c | 9 +++++++-- src/lib-imap-urlauth/imap-urlauth-backend.c | 29 +++++++++++++++++++++++------ src/lib-imap-urlauth/imap-urlauth-backend.h | 4 +++- src/lib-imap-urlauth/imap-urlauth.c | 12 ++++++------ 4 files changed, 39 insertions(+), 15 deletions(-) diffs (152 lines): diff -r dbefcfb9d330 -r 327dc9a8decc src/imap/cmd-resetkey.c --- a/src/imap/cmd-resetkey.c Sat Oct 13 01:38:48 2012 +0300 +++ b/src/imap/cmd-resetkey.c Sat Oct 13 01:57:24 2012 +0300 @@ -24,6 +24,7 @@ struct mail_namespace *ns; enum mailbox_flags flags = MAILBOX_FLAG_READONLY; struct mailbox *box; + int ret; /* check mechanism arguments (we support only INTERNAL mechanism) */ while (!IMAP_ARG_IS_EOL(mech_args)) { @@ -65,8 +66,12 @@ } /* check urlauth environment and reset requested key */ - if (imap_urlauth_reset_mailbox_key(cmd->client->urlauth_ctx, box) < 0) { - client_send_internal_error(cmd); + ret = imap_urlauth_reset_mailbox_key(cmd->client->urlauth_ctx, box); + if (ret <= 0) { + if (ret < 0) + client_send_internal_error(cmd); + else + client_send_storage_error(cmd, mailbox_get_storage(box)); mailbox_free(&box); return TRUE; } diff -r dbefcfb9d330 -r 327dc9a8decc src/lib-imap-urlauth/imap-urlauth-backend.c --- a/src/lib-imap-urlauth/imap-urlauth-backend.c Sat Oct 13 01:38:48 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-backend.c Sat Oct 13 01:57:24 2012 +0300 @@ -88,13 +88,25 @@ int imap_urlauth_backend_get_mailbox_key(struct imap_urlauth_backend *backend, struct mailbox *box, bool create, - unsigned char mailbox_key[IMAP_URLAUTH_KEY_LEN]) + unsigned char mailbox_key_r[IMAP_URLAUTH_KEY_LEN], + const char **error_r, + enum mail_error *error_code_r) { const char *path, *mailbox_key_hex = NULL; + struct mailbox_metadata metadata; const char *mailbox = mailbox_get_vname(box); buffer_t key_buf; int ret; + *error_r = "Internal server error"; + *error_code_r = MAIL_ERROR_TEMP; + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { + *error_r = mailbox_get_last_error(box, error_code_r); + return -1; + } + mailbox = guid_128_to_string(metadata.guid); + path = t_strconcat(IMAP_URLAUTH_PATH, dict_escape_string(mailbox), NULL); if ((ret = imap_urlauth_backend_get_key(backend, path, &mailbox_key_hex)) < 0) @@ -110,8 +122,8 @@ return 0; /* create new key */ - random_fill(mailbox_key, IMAP_URLAUTH_KEY_LEN); - mailbox_key_hex = binary_to_hex(mailbox_key, + random_fill(mailbox_key_r, IMAP_URLAUTH_KEY_LEN); + mailbox_key_hex = binary_to_hex(mailbox_key_r, IMAP_URLAUTH_KEY_LEN); if ((ret = imap_urlauth_backend_set_key(backend, path, mailbox_key_hex)) < 0) @@ -122,7 +134,7 @@ } } else { /* read existing key */ - buffer_create_from_data(&key_buf, mailbox_key, + buffer_create_from_data(&key_buf, mailbox_key_r, IMAP_URLAUTH_KEY_LEN); if (strlen(mailbox_key_hex) != 2*IMAP_URLAUTH_KEY_LEN || hex_to_binary(mailbox_key_hex, &key_buf) < 0 || @@ -131,7 +143,7 @@ mailbox, path); return -1; } - memcpy(mailbox_key, key_buf.data, IMAP_URLAUTH_KEY_LEN); + memcpy(mailbox_key_r, key_buf.data, IMAP_URLAUTH_KEY_LEN); } return 1; } @@ -139,7 +151,12 @@ int imap_urlauth_backend_reset_mailbox_key(struct imap_urlauth_backend *backend, struct mailbox *box) { - const char *path, *mailbox = mailbox_get_vname(box); + const char *path, *mailbox; + struct mailbox_metadata metadata; + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) + return 0; + mailbox = guid_128_to_string(metadata.guid); path = t_strconcat(IMAP_URLAUTH_PATH, dict_escape_string(mailbox), NULL); return imap_urlauth_backend_reset_key(backend, path) < 0 ? -1 : 1; diff -r dbefcfb9d330 -r 327dc9a8decc src/lib-imap-urlauth/imap-urlauth-backend.h --- a/src/lib-imap-urlauth/imap-urlauth-backend.h Sat Oct 13 01:38:48 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-backend.h Sat Oct 13 01:57:24 2012 +0300 @@ -11,7 +11,9 @@ int imap_urlauth_backend_get_mailbox_key(struct imap_urlauth_backend *backend, struct mailbox *box, bool create, - unsigned char mailbox_key[IMAP_URLAUTH_KEY_LEN]); + unsigned char mailbox_key_r[IMAP_URLAUTH_KEY_LEN], + const char **error_r, + enum mail_error *error_code_r); int imap_urlauth_backend_reset_mailbox_key(struct imap_urlauth_backend *backend, struct mailbox *box); int imap_urlauth_backend_reset_all_keys(struct imap_urlauth_backend *backend); diff -r dbefcfb9d330 -r 327dc9a8decc src/lib-imap-urlauth/imap-urlauth.c --- a/src/lib-imap-urlauth/imap-urlauth.c Sat Oct 13 01:38:48 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth.c Sat Oct 13 01:57:24 2012 +0300 @@ -238,6 +238,7 @@ struct imap_msgpart_url *mpurl; struct mailbox *box; const char *error; + enum mail_error error_code; unsigned char mailbox_key[IMAP_URLAUTH_KEY_LEN]; const unsigned char *token; size_t token_len; @@ -298,11 +299,11 @@ /* obtain mailbox key */ ret = imap_urlauth_backend_get_mailbox_key(uctx->backend, box, TRUE, - mailbox_key); + mailbox_key, error_r, + &error_code); if (ret < 0) { - *error_r = "Internal server error"; imap_msgpart_url_free(&mpurl); - return -1; + return ret; } token = imap_urlauth_internal_generate(rumpurl, mailbox_key, &token_len); @@ -418,10 +419,9 @@ /* obtain mailbox key */ ret = imap_urlauth_backend_get_mailbox_key(uctx->backend, box, FALSE, - mailbox_key); + mailbox_key, error_r, + error_code_r); if (ret < 0) { - *error_r = "Internal server error"; - *error_code_r = MAIL_ERROR_TEMP; imap_msgpart_url_free(&mpurl); return -1; } From dovecot at dovecot.org Sat Oct 13 04:28:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 04:28:25 +0300 Subject: dovecot-2.2: lib-dict: Added DICT_ITERATE_FLAG_NO_VALUE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e7f10acf65e3 changeset: 15222:e7f10acf65e3 user: Timo Sirainen date: Sat Oct 13 04:17:45 2012 +0300 description: lib-dict: Added DICT_ITERATE_FLAG_NO_VALUE This allows iterating dictionary without wasting extra time on returning values that aren't needed. diffstat: src/lib-dict/dict-sql.c | 9 +++++++-- src/lib-dict/dict.h | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diffs (39 lines): diff -r 327dc9a8decc -r e7f10acf65e3 src/lib-dict/dict-sql.c --- a/src/lib-dict/dict-sql.c Sat Oct 13 01:57:24 2012 +0300 +++ b/src/lib-dict/dict-sql.c Sat Oct 13 04:17:45 2012 +0300 @@ -356,7 +356,10 @@ T_BEGIN { string_t *query = t_str_new(256); - str_printfa(query, "SELECT %s", map->value_field); + str_append(query, "SELECT "); + if ((ctx->flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + str_printfa(query, "%s,", map->value_field); + /* get all missing fields */ sql_fields = array_get(&map->sql_fields, &count); i = array_count(&values); @@ -367,7 +370,9 @@ i--; } for (; i < count; i++) - str_printfa(query, ",%s", sql_fields[i]); + str_printfa(query, "%s,", sql_fields[i]); + str_truncate(query, str_len(query)-1); + str_printfa(query, " FROM %s", map->table); recurse_type = (ctx->flags & DICT_ITERATE_FLAG_RECURSE) == 0 ? diff -r 327dc9a8decc -r e7f10acf65e3 src/lib-dict/dict.h --- a/src/lib-dict/dict.h Sat Oct 13 01:57:24 2012 +0300 +++ b/src/lib-dict/dict.h Sat Oct 13 04:17:45 2012 +0300 @@ -9,7 +9,8 @@ enum dict_iterate_flags { DICT_ITERATE_FLAG_RECURSE = 0x01, DICT_ITERATE_FLAG_SORT_BY_KEY = 0x02, - DICT_ITERATE_FLAG_SORT_BY_VALUE = 0x04 + DICT_ITERATE_FLAG_SORT_BY_VALUE = 0x04, + DICT_ITERATE_FLAG_NO_VALUE = 0x08 }; enum dict_data_type { From dovecot at dovecot.org Sat Oct 13 04:28:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 04:28:25 +0300 Subject: dovecot-2.2: dict: If DICT_ITERATE_FLAG_NO_VALUE is set, don't s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc3a95fe330b changeset: 15223:dc3a95fe330b user: Timo Sirainen date: Sat Oct 13 04:18:26 2012 +0300 description: dict: If DICT_ITERATE_FLAG_NO_VALUE is set, don't send values to dict client. diffstat: src/dict/dict-commands.c | 7 +++++-- src/dict/dict-connection.h | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diffs (35 lines): diff -r e7f10acf65e3 -r dc3a95fe330b src/dict/dict-commands.c --- a/src/dict/dict-commands.c Sat Oct 13 04:17:45 2012 +0300 +++ b/src/dict/dict-commands.c Sat Oct 13 04:18:26 2012 +0300 @@ -53,8 +53,10 @@ o_stream_cork(conn->output); while (dict_iterate(conn->iter_ctx, &key, &value)) { str_truncate(str, 0); - str_printfa(str, "%c%s\t%s\n", DICT_PROTOCOL_REPLY_OK, - key, value); + str_printfa(str, "%c%s\t", DICT_PROTOCOL_REPLY_OK, key); + if ((conn->iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + str_append(str, value); + str_append_c(str, '\n'); o_stream_nsend(conn->output, str_data(str), str_len(str)); if (o_stream_get_buffer_used_size(conn->output) > @@ -99,6 +101,7 @@ /* */ conn->iter_ctx = dict_iterate_init_multiple(conn->dict, args+1, flags); + conn->iter_flags = flags; o_stream_set_flush_callback(conn->output, cmd_iterate_flush, conn); (void)cmd_iterate_flush(conn); diff -r e7f10acf65e3 -r dc3a95fe330b src/dict/dict-connection.h --- a/src/dict/dict-connection.h Sat Oct 13 04:17:45 2012 +0300 +++ b/src/dict/dict-connection.h Sat Oct 13 04:18:26 2012 +0300 @@ -24,6 +24,7 @@ struct ostream *output; struct dict_iterate_context *iter_ctx; + enum dict_iterate_flags iter_flags; /* There are only a few transactions per client, so keeping them in array is fast enough */ From dovecot at dovecot.org Sat Oct 13 04:28:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 04:28:25 +0300 Subject: dovecot-2.2: lib-storage: Added support to get/set/iterate mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5659c178bdeb changeset: 15224:5659c178bdeb user: Timo Sirainen date: Sat Oct 13 04:27:14 2012 +0300 description: lib-storage: Added support to get/set/iterate mailbox attributes. The attributes are stored in a dict specified by mail_attribute_dict setting. The idea is to use this as storage for extensions that require per-mailbox key=value pairs, such as METADATA. diffstat: src/lib-storage/fail-mailbox.c | 5 +++ src/lib-storage/index/Makefile.am | 2 + src/lib-storage/index/cydir/cydir-storage.c | 5 +++ src/lib-storage/index/dbox-multi/mdbox-storage.c | 5 +++ src/lib-storage/index/dbox-single/sdbox-storage.c | 5 +++ src/lib-storage/index/imapc/imapc-storage.c | 5 +++ src/lib-storage/index/index-storage.c | 26 +++++++++++++++ src/lib-storage/index/index-storage.h | 14 ++++++++ src/lib-storage/index/maildir/maildir-storage.c | 5 +++ src/lib-storage/index/mbox/mbox-storage.c | 5 +++ src/lib-storage/index/pop3c/pop3c-storage.c | 5 +++ src/lib-storage/index/raw/raw-storage.c | 5 +++ src/lib-storage/mail-storage-private.h | 19 +++++++++++ src/lib-storage/mail-storage-settings.c | 2 + src/lib-storage/mail-storage-settings.h | 1 + src/lib-storage/mail-storage.c | 38 +++++++++++++++++++++++ src/lib-storage/mail-storage.h | 25 +++++++++++++++ src/plugins/virtual/virtual-storage.c | 5 +++ 18 files changed, 177 insertions(+), 0 deletions(-) diffs (truncated from 420 to 300 lines): diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/fail-mailbox.c --- a/src/lib-storage/fail-mailbox.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/fail-mailbox.c Sat Oct 13 04:27:14 2012 +0300 @@ -264,6 +264,11 @@ fail_mailbox_set_subscribed, NULL, NULL, + NULL, + NULL, + NULL, + NULL, + NULL, fail_mailbox_sync_init, fail_mailbox_sync_next, fail_mailbox_sync_deinit, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/Makefile.am --- a/src/lib-storage/index/Makefile.am Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/Makefile.am Sat Oct 13 04:27:14 2012 +0300 @@ -5,6 +5,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-imap \ @@ -14,6 +15,7 @@ libstorage_index_la_SOURCES = \ istream-mail.c \ index-attachment.c \ + index-attribute.c \ index-mail.c \ index-mail-binary.c \ index-mail-headers.c \ diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/cydir/cydir-storage.c --- a/src/lib-storage/index/cydir/cydir-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/cydir/cydir-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -141,6 +141,11 @@ index_storage_get_status, NULL, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, cydir_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -440,6 +440,11 @@ index_storage_get_status, mdbox_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, mdbox_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -415,6 +415,11 @@ index_storage_get_status, sdbox_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, sdbox_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -884,6 +884,11 @@ imapc_mailbox_get_status, imapc_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, NULL, NULL, imapc_mailbox_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/index-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -6,6 +6,7 @@ #include "ioloop.h" #include "str.h" #include "mkdir-parents.h" +#include "dict.h" #include "mail-index-alloc-cache.h" #include "mail-index-private.h" #include "mail-index-modseq.h" @@ -359,6 +360,10 @@ void index_storage_mailbox_free(struct mailbox *box) { + if (box->_attr_dict != NULL) { + (void)dict_wait(box->_attr_dict); + dict_deinit(&box->_attr_dict); + } if (box->index_pvt != NULL) mail_index_alloc_cache_unref(&box->index_pvt); if (box->index != NULL) @@ -579,6 +584,23 @@ return mailbox_transaction_commit(&t); } +static int +mailbox_delete_all_attributes(struct mailbox *box, enum mail_attribute_type type) +{ + struct mailbox_attribute_iter *iter; + const char *key; + int ret = 0; + + iter = mailbox_attribute_iter_init(box, type, ""); + while ((key = mailbox_attribute_iter_next(iter)) != NULL) { + if (mailbox_attribute_unset(box, type, key) < 0) + ret = -1; + } + if (mailbox_attribute_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + int index_storage_mailbox_delete(struct mailbox *box) { struct mailbox_metadata metadata; @@ -609,6 +631,10 @@ if (!box->deleting_must_be_empty) { if (mailbox_expunge_all_mails(box) < 0) return -1; + if (mailbox_delete_all_attributes(box, MAIL_ATTRIBUTE_TYPE_PRIVATE) < 0) + return -1; + if (mailbox_delete_all_attributes(box, MAIL_ATTRIBUTE_TYPE_SHARED) < 0) + return -1; } if (mailbox_mark_index_deleted(box, TRUE) < 0) return -1; diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/index-storage.h Sat Oct 13 04:27:14 2012 +0300 @@ -116,6 +116,20 @@ enum mailbox_metadata_items items, struct mailbox_metadata *metadata_r); +int index_storage_attribute_set(struct mailbox *box, + enum mail_attribute_type type, + const char *key, const char *value); +int index_storage_attribute_get(struct mailbox *box, + enum mail_attribute_type type, + const char *key, const char **value_r); +struct mailbox_attribute_iter * +index_storage_attribute_iter_init(struct mailbox *box, + enum mail_attribute_type type, + const char *prefix); +const char * +index_storage_attribute_iter_next(struct mailbox_attribute_iter *iter); +int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *iter); + struct mail_search_context * index_storage_search_init(struct mailbox_transaction_context *t, struct mail_search_args *args, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -678,6 +678,11 @@ index_storage_get_status, maildir_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, maildir_list_index_has_changed, maildir_list_index_update_sync, maildir_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -817,6 +817,11 @@ index_storage_get_status, mbox_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, mbox_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/pop3c/pop3c-storage.c --- a/src/lib-storage/index/pop3c/pop3c-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -250,6 +250,11 @@ index_storage_get_status, index_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, pop3c_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/index/raw/raw-storage.c --- a/src/lib-storage/index/raw/raw-storage.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/index/raw/raw-storage.c Sat Oct 13 04:27:14 2012 +0300 @@ -220,6 +220,11 @@ index_storage_get_status, index_mailbox_get_metadata, index_storage_set_subscribed, + index_storage_attribute_set, + index_storage_attribute_get, + index_storage_attribute_iter_init, + index_storage_attribute_iter_next, + index_storage_attribute_iter_deinit, index_storage_list_index_has_changed, index_storage_list_index_update_sync, raw_storage_sync_init, diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/mail-storage-private.h Sat Oct 13 04:27:14 2012 +0300 @@ -142,6 +142,17 @@ struct mailbox_metadata *metadata_r); int (*set_subscribed)(struct mailbox *box, bool set); + int (*attribute_set)(struct mailbox *box, enum mail_attribute_type type, + const char *key, const char *value); + int (*attribute_get)(struct mailbox *box, enum mail_attribute_type type, + const char *key, const char **value_r); + struct mailbox_attribute_iter * + (*attribute_iter_init)(struct mailbox *box, + enum mail_attribute_type type, + const char *prefix); + const char *(*attribute_iter_next)(struct mailbox_attribute_iter *iter); + int (*attribute_iter_deinit)(struct mailbox_attribute_iter *iter); + /* Lookup sync extension record and figure out if it mailbox has changed since. Returns 1 = yes, 0 = no, -1 = error. */ int (*list_index_has_changed)(struct mailbox *box, @@ -250,6 +261,8 @@ /* Filled lazily when mailbox is opened, use mailbox_get_path() to access it */ const char *_path; + /* Filled lazily by mailbox_attribute_*() */ + struct dict *_attr_dict; /* default vfuncs for new struct mails. */ const struct mail_vfuncs *mail_vfuncs; @@ -307,6 +320,8 @@ unsigned int disallow_new_keywords:1; /* Mailbox has been synced at least once */ unsigned int synced:1; + /* Failed to create attribute dict, don't try again */ + unsigned int attr_dict_failed:1; }; struct mail_vfuncs { @@ -520,6 +535,10 @@ unsigned int *idx; }; +struct mailbox_attribute_iter { + struct mailbox *box; +}; + /* Modules should use do "my_id = mail_storage_module_id++" and use objects' module_contexts[id] for their own purposes. */ extern struct mail_storage_module_register mail_storage_module_register; diff -r dc3a95fe330b -r 5659c178bdeb src/lib-storage/mail-storage-settings.c --- a/src/lib-storage/mail-storage-settings.c Sat Oct 13 04:18:26 2012 +0300 +++ b/src/lib-storage/mail-storage-settings.c Sat Oct 13 04:27:14 2012 +0300 @@ -30,6 +30,7 @@ DEF(SET_STR_VARS, mail_attachment_dir), DEF(SET_STR, mail_attachment_hash), DEF(SET_SIZE, mail_attachment_min_size), + DEF(SET_STR_VARS, mail_attribute_dict), DEF(SET_UINT, mail_prefetch_count), DEF(SET_STR, mail_cache_fields), DEF(SET_STR, mail_never_cache_fields), @@ -61,6 +62,7 @@ .mail_attachment_dir = "", .mail_attachment_hash = "%{sha1}", .mail_attachment_min_size = 1024*128, From dovecot at dovecot.org Sat Oct 13 04:28:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 04:28:25 +0300 Subject: dovecot-2.2: imap-urlauth: Store urlauth keys to the new mailbox... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6e7e62ee07be changeset: 15225:6e7e62ee07be user: Timo Sirainen date: Sat Oct 13 04:28:09 2012 +0300 description: imap-urlauth: Store urlauth keys to the new mailbox attributes. diffstat: doc/example-config/conf.d/20-imap.conf | 3 - src/imap-urlauth/imap-urlauth-worker-settings.c | 2 - src/imap-urlauth/imap-urlauth-worker-settings.h | 1 - src/imap-urlauth/imap-urlauth-worker.c | 16 +- src/imap/imap-client.c | 17 +- src/imap/imap-settings.c | 2 - src/imap/imap-settings.h | 1 - src/lib-imap-urlauth/imap-urlauth-backend.c | 166 +++++------------------ src/lib-imap-urlauth/imap-urlauth-backend.h | 12 +- src/lib-imap-urlauth/imap-urlauth-private.h | 1 - src/lib-imap-urlauth/imap-urlauth.c | 38 ++--- src/lib-imap-urlauth/imap-urlauth.h | 8 +- 12 files changed, 74 insertions(+), 193 deletions(-) diffs (truncated from 541 to 300 lines): diff -r 5659c178bdeb -r 6e7e62ee07be doc/example-config/conf.d/20-imap.conf --- a/doc/example-config/conf.d/20-imap.conf Sat Oct 13 04:27:14 2012 +0300 +++ b/doc/example-config/conf.d/20-imap.conf Sat Oct 13 04:28:09 2012 +0300 @@ -57,8 +57,5 @@ #imap_client_workarounds = } -# Dictionary containing URLAUTH data. -#imap_urlauth_dict = - # Host allowed in URLAUTH URLs sent by client. "*" allows all. #imap_urlauth_host = diff -r 5659c178bdeb -r 6e7e62ee07be src/imap-urlauth/imap-urlauth-worker-settings.c --- a/src/imap-urlauth/imap-urlauth-worker-settings.c Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker-settings.c Sat Oct 13 04:28:09 2012 +0300 @@ -56,7 +56,6 @@ static const struct setting_define imap_urlauth_worker_setting_defines[] = { DEF(SET_BOOL, verbose_proctitle), - DEF(SET_STR_VARS, imap_urlauth_dict), DEF(SET_STR, imap_urlauth_host), DEF(SET_UINT, imap_urlauth_port), @@ -66,7 +65,6 @@ const struct imap_urlauth_worker_settings imap_urlauth_worker_default_settings = { .verbose_proctitle = FALSE, - .imap_urlauth_dict = "", .imap_urlauth_host = "", .imap_urlauth_port = 143 }; diff -r 5659c178bdeb -r 6e7e62ee07be src/imap-urlauth/imap-urlauth-worker-settings.h --- a/src/imap-urlauth/imap-urlauth-worker-settings.h Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker-settings.h Sat Oct 13 04:28:09 2012 +0300 @@ -7,7 +7,6 @@ bool verbose_proctitle; /* imap_urlauth: */ - const char *imap_urlauth_dict; const char *imap_urlauth_host; unsigned int imap_urlauth_port; }; diff -r 5659c178bdeb -r 6e7e62ee07be src/imap-urlauth/imap-urlauth-worker.c --- a/src/imap-urlauth/imap-urlauth-worker.c Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker.c Sat Oct 13 04:28:09 2012 +0300 @@ -634,11 +634,8 @@ } /* initialize urlauth context */ - if (*set->imap_urlauth_host == '\0' || - *set->imap_urlauth_dict == '\0') { - i_error("%s setting is not configured for user %s", - *set->imap_urlauth_host == '\0' ? - "imap_urlauth_host" : "imap_urlauth_dict", + if (*set->imap_urlauth_host == '\0') { + i_error("imap_urlauth_host setting is not configured for user %s", mail_user->username); client_send_line(client, "NO"); client_abort(client, "Session aborted: URLAUTH not configured"); @@ -646,7 +643,6 @@ } memset(&config, 0, sizeof(config)); - config.dict_uri = set->imap_urlauth_dict; config.url_host = set->imap_urlauth_host; config.url_port = set->imap_urlauth_port; config.access_user = client->access_user; @@ -654,13 +650,7 @@ config.access_applications = (const void *)array_get(&client->access_apps, &count); - if (imap_urlauth_init(client->mail_user, &config, &client->urlauth_ctx) < 0) { - client_send_line(client, "NO"); - client_abort(client, - "Session aborted: Failed to init URLAUTH context"); - return 0; - } - + client->urlauth_ctx = imap_urlauth_init(client->mail_user, &config); if (client->debug) { i_debug("Providing access to user account `%s' on behalf of `%s'", mail_user->username, client->access_user); diff -r 5659c178bdeb -r 6e7e62ee07be src/imap/imap-client.c --- a/src/imap/imap-client.c Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap/imap-client.c Sat Oct 13 04:28:09 2012 +0300 @@ -39,12 +39,11 @@ client_destroy(client, "Disconnected for inactivity"); } -static int client_init_urlauth(struct client *client) +static void client_init_urlauth(struct client *client) { struct imap_urlauth_config config; memset(&config, 0, sizeof(config)); - config.dict_uri = client->set->imap_urlauth_dict; config.url_host = client->set->imap_urlauth_host; config.url_port = client->set->imap_urlauth_port; config.socket_path = t_strconcat(client->user->set->base_dir, @@ -53,7 +52,7 @@ config.access_anonymous = client->user->anonymous; config.access_user = client->user->username; - return imap_urlauth_init(client->user, &config, &client->urlauth_ctx); + client->urlauth_ctx = imap_urlauth_init(client->user, &config); } struct client *client_create(int fd_in, int fd_out, const char *session_id, @@ -129,13 +128,13 @@ str_append(client->capability_string, " NOTIFY"); } - if (*set->imap_urlauth_host != '\0' && *set->imap_urlauth_dict != '\0') { - if (client_init_urlauth(client) == 0 && - !explicit_capability) { - /* Enable URLAUTH capability only when dict is - configured correctly */ + if (*set->imap_urlauth_host != '\0' && + *mail_set->mail_attribute_dict != '\0') { + /* Enable URLAUTH capability only when dict is + configured correctly */ + client_init_urlauth(client); + if (!explicit_capability) str_append(client->capability_string, " URLAUTH URLAUTH=BINARY"); - } } ident = mail_user_get_anvil_userip_ident(client->user); diff -r 5659c178bdeb -r 6e7e62ee07be src/imap/imap-settings.c --- a/src/imap/imap-settings.c Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap/imap-settings.c Sat Oct 13 04:28:09 2012 +0300 @@ -70,7 +70,6 @@ DEF(SET_STR, imap_id_send), DEF(SET_STR, imap_id_log), - DEF(SET_STR_VARS, imap_urlauth_dict), DEF(SET_STR, imap_urlauth_host), DEF(SET_UINT, imap_urlauth_port), @@ -91,7 +90,6 @@ .imap_id_send = "", .imap_id_log = "", - .imap_urlauth_dict = "", .imap_urlauth_host = "", .imap_urlauth_port = 143 }; diff -r 5659c178bdeb -r 6e7e62ee07be src/imap/imap-settings.h --- a/src/imap/imap-settings.h Sat Oct 13 04:27:14 2012 +0300 +++ b/src/imap/imap-settings.h Sat Oct 13 04:28:09 2012 +0300 @@ -24,7 +24,6 @@ const char *imap_id_log; /* imap urlauth: */ - const char *imap_urlauth_dict; const char *imap_urlauth_host; unsigned int imap_urlauth_port; diff -r 5659c178bdeb -r 6e7e62ee07be src/lib-imap-urlauth/imap-urlauth-backend.c --- a/src/lib-imap-urlauth/imap-urlauth-backend.c Sat Oct 13 04:27:14 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-backend.c Sat Oct 13 04:28:09 2012 +0300 @@ -4,117 +4,35 @@ #include "buffer.h" #include "hex-binary.h" #include "randgen.h" -#include "dict.h" #include "mail-user.h" #include "mail-storage.h" +#include "mailbox-list-iter.h" #include "imap-urlauth-private.h" #include "imap-urlauth-backend.h" -#define IMAP_URLAUTH_PATH DICT_PATH_PRIVATE"imap-urlauth/" +#define IMAP_URLAUTH_KEY "imap-urlauth" -struct imap_urlauth_backend { - struct mail_user *user; - struct dict *dict; -}; - -int imap_urlauth_backend_create(struct mail_user *user, const char *dict_uri, - struct imap_urlauth_backend **backend_r) -{ - struct imap_urlauth_backend *backend; - struct dict *dict; - const char *error; - - if (user->mail_debug) - i_debug("imap-urlauth backend: opening backend dict URI %s", dict_uri); - - if (dict_init(dict_uri, DICT_DATA_TYPE_STRING, - user->username, user->set->base_dir, &dict, &error) < 0) { - i_error("imap_urlauth_dict: Failed to initialize dict: %s", error); - return -1; - } - - backend = i_new(struct imap_urlauth_backend, 1); - backend->user = user; - backend->dict = dict; - - random_init(); - *backend_r = backend; - return 0; -} - -void imap_urlauth_backend_destroy(struct imap_urlauth_backend **_backend) -{ - struct imap_urlauth_backend *backend = *_backend; - - *_backend = NULL; - - if (backend->dict != NULL) { - (void)dict_wait(backend->dict); - dict_deinit(&backend->dict); - } - i_free(backend); - random_deinit(); -} - -static int -imap_urlauth_backend_set_key(struct imap_urlauth_backend *backend, - const char *path, const char *mailbox_key) -{ - struct dict_transaction_context *dtrans; - - dtrans = dict_transaction_begin(backend->dict); - dict_set(dtrans, path, mailbox_key); - return dict_transaction_commit(&dtrans) < 0 ? -1 : 1; -} - -static int -imap_urlauth_backend_reset_key(struct imap_urlauth_backend *backend, - const char *path) -{ - struct dict_transaction_context *dtrans; - - dtrans = dict_transaction_begin(backend->dict); - dict_unset(dtrans, path); - return dict_transaction_commit(&dtrans) < 0 ? -1 : 1; -} - -static int -imap_urlauth_backend_get_key(struct imap_urlauth_backend *backend, - const char *path, const char **mailbox_key_r) -{ - return dict_lookup(backend->dict, pool_datastack_create(), path, - mailbox_key_r); -} - -int imap_urlauth_backend_get_mailbox_key(struct imap_urlauth_backend *backend, - struct mailbox *box, bool create, +int imap_urlauth_backend_get_mailbox_key(struct mailbox *box, bool create, unsigned char mailbox_key_r[IMAP_URLAUTH_KEY_LEN], const char **error_r, enum mail_error *error_code_r) { - const char *path, *mailbox_key_hex = NULL; - struct mailbox_metadata metadata; - const char *mailbox = mailbox_get_vname(box); + struct mail_user *user = mail_storage_get_user(mailbox_get_storage(box)); + const char *mailbox_key_hex = NULL; buffer_t key_buf; int ret; *error_r = "Internal server error"; *error_code_r = MAIL_ERROR_TEMP; - if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { - *error_r = mailbox_get_last_error(box, error_code_r); - return -1; - } - mailbox = guid_128_to_string(metadata.guid); - - path = t_strconcat(IMAP_URLAUTH_PATH, dict_escape_string(mailbox), NULL); - if ((ret = imap_urlauth_backend_get_key(backend, path, - &mailbox_key_hex)) < 0) + ret = mailbox_attribute_get(box, MAIL_ATTRIBUTE_TYPE_PRIVATE, + IMAP_URLAUTH_KEY, &mailbox_key_hex); + if (ret < 0) return -1; - if (backend->user->mail_debug) { - i_debug("imap-urlauth backend: %skey found for mailbox %s at %s", - (ret > 0 ? "" : "no "), mailbox, path); + if (user->mail_debug) { + i_debug("imap-urlauth: %skey found for mailbox %s", + (ret > 0 ? "" : "no "), mailbox_get_vname(box)); } if (ret == 0) { @@ -125,12 +43,13 @@ random_fill(mailbox_key_r, IMAP_URLAUTH_KEY_LEN); mailbox_key_hex = binary_to_hex(mailbox_key_r, IMAP_URLAUTH_KEY_LEN); - if ((ret = imap_urlauth_backend_set_key(backend, path, - mailbox_key_hex)) < 0) + ret = mailbox_attribute_set(box, MAIL_ATTRIBUTE_TYPE_PRIVATE, + IMAP_URLAUTH_KEY, mailbox_key_hex); + if (ret < 0) return -1; From dovecot at dovecot.org Sat Oct 13 04:30:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Oct 2012 04:30:25 +0300 Subject: dovecot-2.2: lib-storage: Added missing file from previous mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0bb400d84134 changeset: 15226:0bb400d84134 user: Timo Sirainen date: Sat Oct 13 04:30:17 2012 +0300 description: lib-storage: Added missing file from previous mailbox attribute commit. diffstat: src/lib-storage/index/index-attribute.c | 168 ++++++++++++++++++++++++++++++++ 1 files changed, 168 insertions(+), 0 deletions(-) diffs (172 lines): diff -r 6e7e62ee07be -r 0bb400d84134 src/lib-storage/index/index-attribute.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-attribute.c Sat Oct 13 04:30:17 2012 +0300 @@ -0,0 +1,168 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "index-storage.h" + +#define KEY_PREFIX_PRIVATE "priv/" +#define KEY_PREFIX_SHARED "shared/" + +struct index_storage_attribute_iter { + struct mailbox_attribute_iter iter; + struct dict_iterate_context *diter; + char *prefix; + unsigned int prefix_len; +}; + +static int index_storage_get_dict(struct mailbox *box, struct dict **dict_r, + const char **mailbox_prefix_r) +{ + struct mailbox_metadata metadata; + const char *error; + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) + return -1; + *mailbox_prefix_r = guid_128_to_string(metadata.guid); + + if (box->_attr_dict != NULL) { + *dict_r = box->_attr_dict; + return 0; + } + if (*box->storage->set->mail_attribute_dict == '\0') { + mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not enabled"); + return -1; + } + if (box->attr_dict_failed) { + mail_storage_set_internal_error(box->storage); + return -1; + } + + if (dict_init(box->storage->set->mail_attribute_dict, + DICT_DATA_TYPE_STRING, + box->storage->user->username, + box->storage->user->set->base_dir, + &box->_attr_dict, &error) < 0) { + mail_storage_set_critical(box->storage, + "mail_attribute_dict: dict_init(%s) failed: %s", + box->storage->set->mail_attribute_dict, error); + return -1; + } + *dict_r = box->_attr_dict; + return 0; +} + +static const char * +key_get_prefixed(enum mail_attribute_type type, const char *mailbox_prefix, + const char *key) +{ + switch (type) { + case MAIL_ATTRIBUTE_TYPE_PRIVATE: + return t_strconcat(KEY_PREFIX_PRIVATE, mailbox_prefix, "/", + key, NULL); + case MAIL_ATTRIBUTE_TYPE_SHARED: + return t_strconcat(KEY_PREFIX_SHARED, mailbox_prefix, "/", + key, NULL); + } + i_unreached(); +} + +int index_storage_attribute_set(struct mailbox *box, + enum mail_attribute_type type, + const char *key, const char *value) +{ + struct dict_transaction_context *dtrans; + struct dict *dict; + const char *mailbox_prefix; + + if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) + return -1; + + T_BEGIN { + key = key_get_prefixed(type, mailbox_prefix, key); + dtrans = dict_transaction_begin(dict); + if (value != NULL) + dict_set(dtrans, key, value); + else + dict_unset(dtrans, key); + } T_END; + if (dict_transaction_commit(&dtrans) < 0) { + mail_storage_set_internal_error(box->storage); + return -1; + } + return 0; +} + +int index_storage_attribute_get(struct mailbox *box, + enum mail_attribute_type type, + const char *key, const char **value_r) +{ + struct dict *dict; + const char *mailbox_prefix; + int ret; + + if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) + return -1; + + ret = dict_lookup(dict, pool_datastack_create(), + key_get_prefixed(type, mailbox_prefix, key), value_r); + if (ret < 0) { + mail_storage_set_internal_error(box->storage); + return -1; + } + if (ret == 0) + *value_r = NULL; + return ret; +} + +struct mailbox_attribute_iter * +index_storage_attribute_iter_init(struct mailbox *box, + enum mail_attribute_type type, + const char *prefix) +{ + struct index_storage_attribute_iter *iter; + struct dict *dict; + const char *mailbox_prefix; + + iter = i_new(struct index_storage_attribute_iter, 1); + iter->iter.box = box; + if (index_storage_get_dict(box, &dict, &mailbox_prefix) == 0) { + iter->prefix = i_strdup(key_get_prefixed(type, mailbox_prefix, + prefix)); + iter->prefix_len = strlen(iter->prefix); + iter->diter = dict_iterate_init(dict, iter->prefix, + DICT_ITERATE_FLAG_RECURSE | + DICT_ITERATE_FLAG_NO_VALUE); + } + return &iter->iter; +} + +const char * +index_storage_attribute_iter_next(struct mailbox_attribute_iter *_iter) +{ + struct index_storage_attribute_iter *iter = + (struct index_storage_attribute_iter *)_iter; + const char *key, *value; + + if (iter->diter == NULL || !dict_iterate(iter->diter, &key, &value)) + return NULL; + + i_assert(strncmp(key, iter->prefix, iter->prefix_len) == 0); + key += iter->prefix_len; + return key; +} + +int index_storage_attribute_iter_deinit(struct mailbox_attribute_iter *_iter) +{ + struct index_storage_attribute_iter *iter = + (struct index_storage_attribute_iter *)_iter; + int ret; + + ret = iter->diter == NULL ? -1 : + dict_iterate_deinit(&iter->diter); + if (ret < 0) + mail_storage_set_internal_error(_iter->box->storage); + i_free(iter->prefix); + i_free(iter); + return ret; +} From pigeonhole at rename-it.nl Sat Oct 13 11:31:03 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:31:03 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: dict script: fixed potential ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/2b5ff3818a9f changeset: 1659:2b5ff3818a9f user: Stephan Bosch date: Sat Oct 13 10:30:57 2012 +0200 description: lib-sieve: dict script: fixed potential segfault occuring when dict initialization fails. diffstat: src/lib-sieve/sieve-script-dict.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r ceef02768dee -r 2b5ff3818a9f src/lib-sieve/sieve-script-dict.c --- a/src/lib-sieve/sieve-script-dict.c Thu Oct 11 22:07:50 2012 +0200 +++ b/src/lib-sieve/sieve-script-dict.c Sat Oct 13 10:30:57 2012 +0200 @@ -112,7 +112,6 @@ "sieve dict backend: failed to initialize dict with data `%s' " "for user `%s'", data, username); *error_r = SIEVE_ERROR_TEMP_FAIL; - dict_deinit(&script->dict); return -1; } From pigeonhole at rename-it.nl Sat Oct 13 11:36:20 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:20 +0200 Subject: dovecot-2.2-pigeonhole: testsuite: date extension: Added test fo... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1963a329c539 changeset: 1691:1963a329c539 user: Stephan Bosch date: Wed Sep 26 21:52:32 2012 +0200 description: testsuite: date extension: Added test for date comparisons. diffstat: tests/extensions/date/basic.svtest | 33 +++++++++++++++++++++++++++++++++ 1 files changed, 33 insertions(+), 0 deletions(-) diffs (40 lines): diff -r bf7b7aa7f667 -r 1963a329c539 tests/extensions/date/basic.svtest --- a/tests/extensions/date/basic.svtest Wed Sep 26 21:51:46 2012 +0200 +++ b/tests/extensions/date/basic.svtest Wed Sep 26 21:52:32 2012 +0200 @@ -39,3 +39,36 @@ test_fail "matched invalid date: ${0}"; } } + +test "Comparison" { + if not date :is "delivery-date" "date" "2009-07-22" { + if date :matches "delivery-date" "date" "*" { set "date" "${1}"; } + test_fail "date is invalid: ${date}"; + } + + if not date :value "ge" "delivery-date" "date" "2009-07-22" { + test_fail "date comparison ge failed equal"; + } + + if not date :value "ge" "delivery-date" "date" "2009-07-21" { + test_fail "date comparison ge failed greater"; + } + + if anyof (not date :value "ge" "delivery-date" "date" "2009-06-22", + not date :value "ge" "date" "date" "2006-07-22" ) { + test_fail "date comparison ge failed much greater"; + } + + if not date :value "le" "delivery-date" "date" "2009-07-22" { + test_fail "date comparison le failed equal"; + } + + if not date :value "le" "delivery-date" "date" "2009-07-23" { + test_fail "date comparison le failed less"; + } + + if anyof (not date :value "le" "delivery-date" "date" "2009-09-22", + not date :value "le" "date" "date" "2012-07-22" ) { + test_fail "date comparison ge failed much less"; + } +} From pigeonhole at rename-it.nl Sat Oct 13 11:36:21 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:21 +0200 Subject: dovecot-2.2-pigeonhole: ManageSieve: fixed handling of unkown co... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ceef02768dee changeset: 1694:ceef02768dee user: Stephan Bosch date: Thu Oct 11 22:07:50 2012 +0200 description: ManageSieve: fixed handling of unkown commands pre-login. Forgot to skip line upon error. diffstat: src/managesieve-login/client.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 5c1ce25596ed -r ceef02768dee src/managesieve-login/client.c --- a/src/managesieve-login/client.c Thu Oct 11 20:26:03 2012 +0200 +++ b/src/managesieve-login/client.c Thu Oct 11 22:07:50 2012 +0200 @@ -195,6 +195,8 @@ if ( cmd->name != NULL ) client->cmd = cmd; + else + client->skip_line = TRUE; } if ( client->cmd != NULL && !client->cmd_parsed_args ) { From pigeonhole at rename-it.nl Sat Oct 13 11:36:21 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:21 +0200 Subject: dovecot-2.2-pigeonhole: Adjusted to Dovecot dict API changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/6979dbc334fa changeset: 1697:6979dbc334fa user: Stephan Bosch date: Sat Oct 13 10:36:11 2012 +0200 description: Adjusted to Dovecot dict API changes. diffstat: src/lib-sieve/sieve-script-dict.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (26 lines): diff -r 9f9459bd038f -r 6979dbc334fa src/lib-sieve/sieve-script-dict.c --- a/src/lib-sieve/sieve-script-dict.c Sat Oct 13 10:35:19 2012 +0200 +++ b/src/lib-sieve/sieve-script-dict.c Sat Oct 13 10:36:11 2012 +0200 @@ -68,7 +68,7 @@ struct sieve_instance *svinst = _script->svinst; struct sieve_error_handler *ehandler = _script->ehandler; const char *username = NULL, *name = _script->name; - const char *path; + const char *path, *error; int ret; if ( options != NULL ) { @@ -117,11 +117,11 @@ script->dict_uri = p_strdup(_script->pool, data); ret = dict_init(script->dict_uri, DICT_DATA_TYPE_STRING, username, - svinst->base_dir, &script->dict); + svinst->base_dir, &script->dict, &error); if ( ret < 0 ) { sieve_critical(svinst, ehandler, name, "failed to open sieve script", "sieve dict backend: failed to initialize dict with data `%s' " - "for user `%s'", data, username); + "for user `%s': error", data, username, error); *error_r = SIEVE_ERROR_TEMP_FAIL; return -1; } From pigeonhole at rename-it.nl Sat Oct 13 11:36:20 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:20 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: fileinto/keep: don't log quot... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7301a7ed8800 changeset: 1692:7301a7ed8800 user: Stephan Bosch date: Thu Oct 11 01:48:29 2012 +0200 description: lib-sieve: fileinto/keep: don't log quota errors to syslog but rather to user's log. Prevents administrator frustration about useless log messages caused by users with too much mail. diffstat: src/lib-sieve/sieve-actions.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 1963a329c539 -r 7301a7ed8800 src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Wed Sep 26 21:52:32 2012 +0200 +++ b/src/lib-sieve/sieve-actions.c Thu Oct 11 01:48:29 2012 +0200 @@ -604,8 +604,9 @@ (mailbox_get_storage(trans->box), &error_code); } - if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS ) - { + if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS && + error_code != MAIL_ERROR_NOSPACE) + { sieve_result_global_error(aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); } else { From pigeonhole at rename-it.nl Sat Oct 13 11:36:20 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:20 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: date extension: Generate warn... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/bf7b7aa7f667 changeset: 1690:bf7b7aa7f667 user: Stephan Bosch date: Wed Sep 26 21:51:46 2012 +0200 description: lib-sieve: date extension: Generate warning when invalid date part is specified. diffstat: src/lib-sieve/plugins/date/ext-date-common.c | 23 +++++++++++++-------- src/lib-sieve/plugins/date/ext-date-common.h | 6 +++- src/lib-sieve/plugins/date/tst-date.c | 30 ++++++++++++++++++++++----- 3 files changed, 42 insertions(+), 17 deletions(-) diffs (175 lines): diff -r fcd5208ed8b2 -r bf7b7aa7f667 src/lib-sieve/plugins/date/ext-date-common.c --- a/src/lib-sieve/plugins/date/ext-date-common.c Sun Sep 23 17:09:26 2012 +0200 +++ b/src/lib-sieve/plugins/date/ext-date-common.c Wed Sep 26 21:51:46 2012 +0200 @@ -282,23 +282,28 @@ unsigned int date_parts_count = N_ELEMENTS(date_parts); -const char *ext_date_part_extract -(const char *part, struct tm *tm, int zone_offset) +const struct ext_date_part *ext_date_part_find(const char *part) { unsigned int i; for ( i = 0; i < date_parts_count; i++ ) { if ( strcasecmp(date_parts[i]->identifier, part) == 0 ) { - if ( date_parts[i]->get_string != NULL ) - return date_parts[i]->get_string(tm, zone_offset); - - return NULL; + return date_parts[i]; } } return NULL; } +const char *ext_date_part_extract +(const struct ext_date_part *dpart, struct tm *tm, int zone_offset) +{ + if ( dpart == NULL || dpart->get_string == NULL ) + return NULL; + + return dpart->get_string(tm, zone_offset); +} + /* * Date part implementations */ @@ -486,7 +491,7 @@ struct sieve_stringlist *field_values; int time_zone; - const char *date_part; + const struct ext_date_part *date_part; time_t local_time; int local_zone; @@ -496,7 +501,7 @@ struct sieve_stringlist *ext_date_stringlist_create (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_values, - int time_zone, const char *date_part) + int time_zone, const struct ext_date_part *dpart) { struct ext_date_stringlist *strlist; @@ -507,7 +512,7 @@ strlist->strlist.reset = ext_date_stringlist_reset; strlist->field_values = field_values; strlist->time_zone = time_zone; - strlist->date_part = date_part; + strlist->date_part = dpart; strlist->local_time = ext_date_get_current_date(renv, &strlist->local_zone); diff -r fcd5208ed8b2 -r bf7b7aa7f667 src/lib-sieve/plugins/date/ext-date-common.h --- a/src/lib-sieve/plugins/date/ext-date-common.h Sun Sep 23 17:09:26 2012 +0200 +++ b/src/lib-sieve/plugins/date/ext-date-common.h Wed Sep 26 21:51:46 2012 +0200 @@ -60,8 +60,10 @@ const char *(*get_string)(struct tm *tm, int zone_offset); }; +const struct ext_date_part *ext_date_part_find(const char *part); + const char *ext_date_part_extract - (const char *part, struct tm *tm, int zone_offset); + (const struct ext_date_part *dpart, struct tm *tm, int zone_offset); /* * Date stringlist @@ -74,7 +76,7 @@ struct sieve_stringlist *ext_date_stringlist_create (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_values, - int time_zone, const char *date_part); + int time_zone, const struct ext_date_part *dpart); diff -r fcd5208ed8b2 -r bf7b7aa7f667 src/lib-sieve/plugins/date/tst-date.c --- a/src/lib-sieve/plugins/date/tst-date.c Sun Sep 23 17:09:26 2012 +0200 +++ b/src/lib-sieve/plugins/date/tst-date.c Wed Sep 26 21:51:46 2012 +0200 @@ -260,7 +260,7 @@ return FALSE; if ( !sieve_command_verify_headers_argument(valdtr, arg) ) - return FALSE; + return FALSE; arg = sieve_ast_argument_next(arg); } @@ -275,6 +275,16 @@ if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) return FALSE; + if ( sieve_argument_is_string_literal(arg) ) { + const char * part = sieve_ast_argument_strc(arg); + + if ( ext_date_part_find(part) == NULL ) { + sieve_argument_validate_warning + (valdtr, arg, "specified date part `%s' is not known", + str_sanitize(part, 80)); + } + } + arg = sieve_ast_argument_next(arg); /* Check key list */ @@ -349,7 +359,7 @@ if ( !sieve_opr_string_dump_ex(denv, address, "zone", "ORIGINAL") ) return FALSE; break; - default: + default: return FALSE; } } @@ -381,6 +391,7 @@ struct sieve_stringlist *hdr_list = NULL, *hdr_value_list; struct sieve_stringlist *value_list, *key_list; bool zone_specified = FALSE, zone_literal = TRUE; + const struct ext_date_part *dpart; int time_zone; int match, ret; @@ -433,11 +444,19 @@ } else if ( !ext_date_parse_timezone(str_c(zone), &time_zone) ) { if ( !zone_literal ) sieve_runtime_warning(renv, NULL, - "specified :zone argument '%s' is not a valid timezone " + "specified :zone argument `%s' is not a valid timezone " "(using local zone)", str_sanitize(str_c(zone), 40)); time_zone = EXT_DATE_TIMEZONE_LOCAL; } + if ( (dpart=ext_date_part_find(str_c(date_part))) == NULL ) { + sieve_runtime_warning(renv, NULL, + "specified date part argument `%s' is not known", + str_sanitize(str_c(date_part), 40)); + sieve_interpreter_set_test_result(renv->interp, FALSE); + return SIEVE_EXEC_OK; + } + /* * Perform test */ @@ -449,7 +468,7 @@ /* Create value stringlist */ hdr_value_list = sieve_message_header_stringlist_create(renv, hdr_list, FALSE); value_list = ext_date_stringlist_create - (renv, hdr_value_list, time_zone, str_c(date_part)); + (renv, hdr_value_list, time_zone, dpart); } else if ( sieve_operation_is(op, currentdate_operation) ) { /* Use time stamp recorded at the time the script first started */ @@ -457,8 +476,7 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "currentdatedate test"); /* Create value stringlist */ - value_list = ext_date_stringlist_create - (renv, NULL, time_zone, str_c(date_part)); + value_list = ext_date_stringlist_create(renv, NULL, time_zone, dpart); } else { i_unreached(); } From pigeonhole at rename-it.nl Sat Oct 13 11:36:21 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:21 +0200 Subject: dovecot-2.2-pigeonhole: Merged changes from Pigeonhole v0.3. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/9f9459bd038f changeset: 1696:9f9459bd038f user: Stephan Bosch date: Sat Oct 13 10:35:19 2012 +0200 description: Merged changes from Pigeonhole v0.3. diffstat: src/lib-sieve/plugins/date/ext-date-common.c | 23 +++++--- src/lib-sieve/plugins/date/ext-date-common.h | 6 +- src/lib-sieve/plugins/date/tst-date.c | 30 ++++++++-- src/lib-sieve/sieve-actions.c | 13 ++- src/lib-sieve/sieve-error-private.h | 3 +- src/lib-sieve/sieve-error.c | 76 +++++++++++++++++++++++++-- src/lib-sieve/sieve-error.h | 17 ++++++ src/lib-sieve/sieve-result.c | 21 +++++++- src/lib-sieve/sieve-result.h | 6 ++ src/lib-sieve/sieve-script-dict.c | 1 - src/managesieve-login/client.c | 2 + src/plugins/lda-sieve/lda-sieve-plugin.c | 18 +++++- tests/extensions/date/basic.svtest | 33 ++++++++++++ 13 files changed, 213 insertions(+), 36 deletions(-) diffs (truncated from 540 to 300 lines): diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/plugins/date/ext-date-common.c --- a/src/lib-sieve/plugins/date/ext-date-common.c Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/plugins/date/ext-date-common.c Sat Oct 13 10:35:19 2012 +0200 @@ -283,23 +283,28 @@ unsigned int date_parts_count = N_ELEMENTS(date_parts); -const char *ext_date_part_extract -(const char *part, struct tm *tm, int zone_offset) +const struct ext_date_part *ext_date_part_find(const char *part) { unsigned int i; for ( i = 0; i < date_parts_count; i++ ) { if ( strcasecmp(date_parts[i]->identifier, part) == 0 ) { - if ( date_parts[i]->get_string != NULL ) - return date_parts[i]->get_string(tm, zone_offset); - - return NULL; + return date_parts[i]; } } return NULL; } +const char *ext_date_part_extract +(const struct ext_date_part *dpart, struct tm *tm, int zone_offset) +{ + if ( dpart == NULL || dpart->get_string == NULL ) + return NULL; + + return dpart->get_string(tm, zone_offset); +} + /* * Date part implementations */ @@ -455,7 +460,7 @@ struct sieve_stringlist *field_values; int time_zone; - const char *date_part; + const struct ext_date_part *date_part; time_t local_time; int local_zone; @@ -465,7 +470,7 @@ struct sieve_stringlist *ext_date_stringlist_create (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_values, - int time_zone, const char *date_part) + int time_zone, const struct ext_date_part *dpart) { struct ext_date_stringlist *strlist; @@ -476,7 +481,7 @@ strlist->strlist.reset = ext_date_stringlist_reset; strlist->field_values = field_values; strlist->time_zone = time_zone; - strlist->date_part = date_part; + strlist->date_part = dpart; strlist->local_time = ext_date_get_current_date(renv, &strlist->local_zone); diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/plugins/date/ext-date-common.h --- a/src/lib-sieve/plugins/date/ext-date-common.h Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/plugins/date/ext-date-common.h Sat Oct 13 10:35:19 2012 +0200 @@ -60,8 +60,10 @@ const char *(*get_string)(struct tm *tm, int zone_offset); }; +const struct ext_date_part *ext_date_part_find(const char *part); + const char *ext_date_part_extract - (const char *part, struct tm *tm, int zone_offset); + (const struct ext_date_part *dpart, struct tm *tm, int zone_offset); /* * Date stringlist @@ -74,7 +76,7 @@ struct sieve_stringlist *ext_date_stringlist_create (const struct sieve_runtime_env *renv, struct sieve_stringlist *field_values, - int time_zone, const char *date_part); + int time_zone, const struct ext_date_part *dpart); diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/plugins/date/tst-date.c --- a/src/lib-sieve/plugins/date/tst-date.c Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/plugins/date/tst-date.c Sat Oct 13 10:35:19 2012 +0200 @@ -260,7 +260,7 @@ return FALSE; if ( !sieve_command_verify_headers_argument(valdtr, arg) ) - return FALSE; + return FALSE; arg = sieve_ast_argument_next(arg); } @@ -275,6 +275,16 @@ if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) return FALSE; + if ( sieve_argument_is_string_literal(arg) ) { + const char * part = sieve_ast_argument_strc(arg); + + if ( ext_date_part_find(part) == NULL ) { + sieve_argument_validate_warning + (valdtr, arg, "specified date part `%s' is not known", + str_sanitize(part, 80)); + } + } + arg = sieve_ast_argument_next(arg); /* Check key list */ @@ -349,7 +359,7 @@ if ( !sieve_opr_string_dump_ex(denv, address, "zone", "ORIGINAL") ) return FALSE; break; - default: + default: return FALSE; } } @@ -381,6 +391,7 @@ struct sieve_stringlist *hdr_list = NULL, *hdr_value_list; struct sieve_stringlist *value_list, *key_list; bool zone_specified = FALSE, zone_literal = TRUE; + const struct ext_date_part *dpart; int time_zone; int match, ret; @@ -433,11 +444,19 @@ } else if ( !ext_date_parse_timezone(str_c(zone), &time_zone) ) { if ( !zone_literal ) sieve_runtime_warning(renv, NULL, - "specified :zone argument '%s' is not a valid timezone " + "specified :zone argument `%s' is not a valid timezone " "(using local zone)", str_sanitize(str_c(zone), 40)); time_zone = EXT_DATE_TIMEZONE_LOCAL; } + if ( (dpart=ext_date_part_find(str_c(date_part))) == NULL ) { + sieve_runtime_warning(renv, NULL, + "specified date part argument `%s' is not known", + str_sanitize(str_c(date_part), 40)); + sieve_interpreter_set_test_result(renv->interp, FALSE); + return SIEVE_EXEC_OK; + } + /* * Perform test */ @@ -449,7 +468,7 @@ /* Create value stringlist */ hdr_value_list = sieve_message_header_stringlist_create(renv, hdr_list, FALSE); value_list = ext_date_stringlist_create - (renv, hdr_value_list, time_zone, str_c(date_part)); + (renv, hdr_value_list, time_zone, dpart); } else if ( sieve_operation_is(op, currentdate_operation) ) { /* Use time stamp recorded at the time the script first started */ @@ -457,8 +476,7 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "currentdatedate test"); /* Create value stringlist */ - value_list = ext_date_stringlist_create - (renv, NULL, time_zone, str_c(date_part)); + value_list = ext_date_stringlist_create(renv, NULL, time_zone, dpart); } else { i_unreached(); } diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/sieve-actions.c Sat Oct 13 10:35:19 2012 +0200 @@ -605,13 +605,16 @@ (mailbox_get_storage(trans->box), &error_code); } - if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS ) - { + if ( error_code == MAIL_ERROR_NOTFOUND || + error_code == MAIL_ERROR_PARAMS ) { + sieve_result_error(aenv, "failed to store into mailbox %s: %s", + mailbox_name, errstr); + } else if ( error_code == MAIL_ERROR_NOSPACE ) { + sieve_result_global_log_error + (aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); + } else { sieve_result_global_error(aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); - } else { - sieve_result_error(aenv, "failed to store into mailbox %s: %s", - mailbox_name, errstr); } /* Store aborted? */ diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/sieve-error-private.h --- a/src/lib-sieve/sieve-error-private.h Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/sieve-error-private.h Sat Oct 13 10:35:19 2012 +0200 @@ -11,7 +11,8 @@ */ enum sieve_error_flags { - SIEVE_ERROR_FLAG_GLOBAL = (1 << 0) + SIEVE_ERROR_FLAG_GLOBAL = (1 << 0), + SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO = (1 << 1), }; /* diff -r cf079ce8368b -r 9f9459bd038f src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Fri Oct 12 19:25:10 2012 +0200 +++ b/src/lib-sieve/sieve-error.c Sat Oct 13 10:35:19 2012 +0200 @@ -84,11 +84,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->verror - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->verror + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || sieve_errors_more_allowed(ehandler) ) { if ( ehandler->verror != NULL ) @@ -111,11 +117,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->vwarning - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->vwarning + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->vwarning != NULL ) ehandler->vwarning(ehandler, flags, location, fmt, args); @@ -140,7 +152,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_info ) { if ( ehandler->vinfo != NULL ) @@ -164,7 +177,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_debug ) { if ( ehandler->vdebug != NULL ) @@ -300,6 +314,24 @@ (svinst, ehandler, SIEVE_ERROR_FLAG_GLOBAL, location, fmt, args); } +void sieve_global_info_verror +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_verror(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + +void sieve_global_info_vwarning +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_vwarning(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + From pigeonhole at rename-it.nl Sat Oct 13 11:36:21 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:21 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: dict script: fixed potential ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/2b5ff3818a9f changeset: 1695:2b5ff3818a9f user: Stephan Bosch date: Sat Oct 13 10:30:57 2012 +0200 description: lib-sieve: dict script: fixed potential segfault occuring when dict initialization fails. diffstat: src/lib-sieve/sieve-script-dict.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r ceef02768dee -r 2b5ff3818a9f src/lib-sieve/sieve-script-dict.c --- a/src/lib-sieve/sieve-script-dict.c Thu Oct 11 22:07:50 2012 +0200 +++ b/src/lib-sieve/sieve-script-dict.c Sat Oct 13 10:30:57 2012 +0200 @@ -112,7 +112,6 @@ "sieve dict backend: failed to initialize dict with data `%s' " "for user `%s'", data, username); *error_r = SIEVE_ERROR_TEMP_FAIL; - dict_deinit(&script->dict); return -1; } From pigeonhole at rename-it.nl Sat Oct 13 11:36:21 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 13 Oct 2012 10:36:21 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Further improved handling of ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/5c1ce25596ed changeset: 1693:5c1ce25596ed user: Stephan Bosch date: Thu Oct 11 20:26:03 2012 +0200 description: lib-sieve: Further improved handling of quota errors. Added means to log user errors/warnings as info in master log. Previous change was inadequate because an error was still logged. diffstat: src/lib-sieve/sieve-actions.c | 14 +++-- src/lib-sieve/sieve-error-private.h | 3 +- src/lib-sieve/sieve-error.c | 76 ++++++++++++++++++++++++++++--- src/lib-sieve/sieve-error.h | 17 +++++++ src/lib-sieve/sieve-result.c | 21 ++++++++- src/lib-sieve/sieve-result.h | 6 ++ src/plugins/lda-sieve/lda-sieve-plugin.c | 18 ++++++- 7 files changed, 136 insertions(+), 19 deletions(-) diffs (truncated from 303 to 300 lines): diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-actions.c Thu Oct 11 20:26:03 2012 +0200 @@ -604,14 +604,16 @@ (mailbox_get_storage(trans->box), &error_code); } - if ( error_code != MAIL_ERROR_NOTFOUND && error_code != MAIL_ERROR_PARAMS && - error_code != MAIL_ERROR_NOSPACE) - { + if ( error_code == MAIL_ERROR_NOTFOUND || + error_code == MAIL_ERROR_PARAMS ) { + sieve_result_error(aenv, "failed to store into mailbox %s: %s", + mailbox_name, errstr); + } else if ( error_code == MAIL_ERROR_NOSPACE ) { + sieve_result_global_log_error + (aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); + } else { sieve_result_global_error(aenv, "failed to store into mailbox %s: %s", mailbox_name, errstr); - } else { - sieve_result_error(aenv, "failed to store into mailbox %s: %s", - mailbox_name, errstr); } /* Store aborted? */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error-private.h --- a/src/lib-sieve/sieve-error-private.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error-private.h Thu Oct 11 20:26:03 2012 +0200 @@ -11,7 +11,8 @@ */ enum sieve_error_flags { - SIEVE_ERROR_FLAG_GLOBAL = (1 << 0) + SIEVE_ERROR_FLAG_GLOBAL = (1 << 0), + SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO = (1 << 1), }; /* diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error.c --- a/src/lib-sieve/sieve-error.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error.c Thu Oct 11 20:26:03 2012 +0200 @@ -84,11 +84,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->verror - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->verror + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || sieve_errors_more_allowed(ehandler) ) { if ( ehandler->verror != NULL ) @@ -111,11 +117,17 @@ VA_COPY(args_copy, args); - svinst->system_ehandler->vwarning - (svinst->system_ehandler, 0, location, fmt, args_copy); + if ( (flags & SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO) != 0 ) { + svinst->system_ehandler->vinfo + (svinst->system_ehandler, 0, location, fmt, args_copy); + } else { + svinst->system_ehandler->vwarning + (svinst->system_ehandler, 0, location, fmt, args_copy); + } } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->vwarning != NULL ) ehandler->vwarning(ehandler, flags, location, fmt, args); @@ -140,7 +152,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_info ) { if ( ehandler->vinfo != NULL ) @@ -164,7 +177,8 @@ (svinst->system_ehandler, 0, location, fmt, args_copy); } - if ( ehandler == NULL ) return; + if ( ehandler == NULL ) + return; if ( ehandler->parent != NULL || ehandler->log_debug ) { if ( ehandler->vdebug != NULL ) @@ -300,6 +314,24 @@ (svinst, ehandler, SIEVE_ERROR_FLAG_GLOBAL, location, fmt, args); } +void sieve_global_info_verror +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_verror(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + +void sieve_global_info_vwarning +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args) +{ + sieve_direct_vwarning(svinst, ehandler, + (SIEVE_ERROR_FLAG_GLOBAL | SIEVE_ERROR_FLAG_GLOBAL_MAX_INFO), + location, fmt, args); +} + void sieve_global_error (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) @@ -342,6 +374,34 @@ va_end(args); } +void sieve_global_info_error +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + + T_BEGIN { + sieve_global_info_verror(svinst, ehandler, location, fmt, args); + } T_END; + + va_end(args); +} + +void sieve_global_info_warning +(struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + + T_BEGIN { + sieve_global_info_vwarning(svinst, ehandler, location, fmt, args); + } T_END; + + va_end(args); +} + /* * Default (user) error functions */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-error.h --- a/src/lib-sieve/sieve-error.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-error.h Thu Oct 11 20:26:03 2012 +0200 @@ -30,6 +30,11 @@ (struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) ATTR_FORMAT(3, 4); +typedef void (*sieve_sys_error_vfunc_t) + (struct sieve_instance *svinst, const char *fmt, va_list args); +typedef void (*sieve_sys_error_func_t) + (struct sieve_instance *svinst, const char *fmt, ...) ATTR_FORMAT(2, 3); + /* * System errors */ @@ -70,6 +75,12 @@ void sieve_global_vinfo (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, va_list args); +void sieve_global_info_verror + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args); +void sieve_global_info_vwarning + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, va_list args); void sieve_global_error (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, @@ -80,6 +91,12 @@ void sieve_global_info (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); +void sieve_global_info_error + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); +void sieve_global_info_warning + (struct sieve_instance *svinst, struct sieve_error_handler *ehandler, + const char *location, const char *fmt, ...) ATTR_FORMAT(4, 5); /* * Main (user) error functions diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-result.c --- a/src/lib-sieve/sieve-result.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-result.c Thu Oct 11 20:26:03 2012 +0200 @@ -272,7 +272,6 @@ va_end(args); } - void sieve_result_log (const struct sieve_action_exec_env *aenv, const char *fmt, ...) { @@ -293,6 +292,26 @@ va_end(args); } +void sieve_result_global_log_error +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + sieve_global_info_verror(aenv->svinst, aenv->ehandler, NULL, fmt, args); + va_end(args); +} + +void sieve_result_global_log_warning +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + sieve_global_info_vwarning(aenv->svinst, aenv->ehandler, NULL, fmt, args); + va_end(args); +} + /* * Result composition */ diff -r 7301a7ed8800 -r 5c1ce25596ed src/lib-sieve/sieve-result.h --- a/src/lib-sieve/sieve-result.h Thu Oct 11 01:48:29 2012 +0200 +++ b/src/lib-sieve/sieve-result.h Thu Oct 11 20:26:03 2012 +0200 @@ -105,6 +105,12 @@ void sieve_result_global_log (const struct sieve_action_exec_env *aenv, const char *fmt, ...) ATTR_FORMAT(2, 3); +void sieve_result_global_log_error +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) + ATTR_FORMAT(2, 3); +void sieve_result_global_log_warning +(const struct sieve_action_exec_env *aenv, const char *fmt, ...) + ATTR_FORMAT(2, 3); /* * Result composition diff -r 7301a7ed8800 -r 5c1ce25596ed src/plugins/lda-sieve/lda-sieve-plugin.c --- a/src/plugins/lda-sieve/lda-sieve-plugin.c Thu Oct 11 01:48:29 2012 +0200 +++ b/src/plugins/lda-sieve/lda-sieve-plugin.c Thu Oct 11 20:26:03 2012 +0200 @@ -353,17 +353,29 @@ (struct lda_sieve_run_context *srctx, struct sieve_script *script, int status) { struct sieve_instance *svinst = srctx->svinst; + struct sieve_exec_status *estatus = srctx->scriptenv->exec_status; const char *userlog_notice = ""; + sieve_sys_error_func_t error_func = sieve_sys_error; int ret; + if ( estatus != NULL && estatus->last_storage != NULL ) { + enum mail_error mail_error; + + mail_storage_get_last_error(estatus->last_storage, &mail_error); + + /* Don't bother administrator too much with benign errors */ + if ( mail_error == MAIL_ERROR_NOSPACE ) + error_func = sieve_sys_info; + } + if ( script == srctx->user_script && srctx->userlog != NULL ) { userlog_notice = t_strdup_printf - (" (user logfile %s may reveal additional details)", srctx->userlog); + (" (user logfile %s should reveal additional details)", srctx->userlog); } switch ( status ) { case SIEVE_EXEC_FAILURE: - sieve_sys_error(svinst, + error_func(svinst, "execution of script %s failed, but implicit keep was successful%s", sieve_script_location(script), userlog_notice); ret = 1; @@ -376,7 +388,7 @@ ret = -1; break; case SIEVE_EXEC_KEEP_FAILED: - sieve_sys_error(svinst, + error_func(svinst, From dovecot at dovecot.org Tue Oct 16 03:08:40 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Oct 2012 03:08:40 +0300 Subject: dovecot-2.1: lib-master: Fixed -i parameter hand... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0262ede193e5 changeset: 14763:0262ede193e5 user: Timo Sirainen date: Tue Oct 16 03:08:21 2012 +0300 description: lib-master: Fixed -i parameter handling. It previously worked only if the default config socket wasn't usable. diffstat: src/lib-master/master-service.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c8d55ba25f39 -r 0262ede193e5 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Fri Oct 12 23:05:43 2012 +0300 +++ b/src/lib-master/master-service.c Tue Oct 16 03:08:21 2012 +0300 @@ -359,6 +359,7 @@ if (!get_instance_config(arg, &path)) i_fatal("Unknown instance name: %s", arg); service->config_path = i_strdup(path); + service->config_path_is_default = FALSE; break; case 'k': service->keep_environment = TRUE; From dovecot at dovecot.org Tue Oct 16 03:35:07 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Oct 2012 03:35:07 +0300 Subject: dovecot-2.1: dict quota: Fixed a potential crash if quota recalc... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/41aac09497ee changeset: 14764:41aac09497ee user: Timo Sirainen date: Tue Oct 16 03:34:51 2012 +0300 description: dict quota: Fixed a potential crash if quota recalculation was triggered at deinit. diffstat: src/plugins/quota/quota-dict.c | 14 ++++++++++---- src/plugins/quota/quota-dirsize.c | 1 + src/plugins/quota/quota-fs.c | 3 ++- src/plugins/quota/quota-maildir.c | 1 + src/plugins/quota/quota-private.h | 2 +- src/plugins/quota/quota-storage.c | 20 ++++++++++++++++++++ 6 files changed, 35 insertions(+), 6 deletions(-) diffs (123 lines): diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-dict.c --- a/src/plugins/quota/quota-dict.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-dict.c Tue Oct 16 03:34:51 2012 +0300 @@ -86,10 +86,8 @@ { struct dict_quota_root *root = (struct dict_quota_root *)_root; - if (root->dict != NULL) { - (void)dict_wait(root->dict); + if (root->dict != NULL) dict_deinit(&root->dict); - } i_free(root); } @@ -208,6 +206,13 @@ return 0; } +static void dict_quota_flush(struct quota_root *_root) +{ + struct dict_quota_root *root = (struct dict_quota_root *)_root; + + (void)dict_wait(root->dict); +} + struct quota_backend quota_backend_dict = { "dict", @@ -221,6 +226,7 @@ dict_quota_root_get_resources, dict_quota_get_resource, dict_quota_update, - NULL + NULL, + dict_quota_flush } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-dirsize.c --- a/src/plugins/quota/quota-dirsize.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-dirsize.c Tue Oct 16 03:34:51 2012 +0300 @@ -219,6 +219,7 @@ dirsize_quota_root_get_resources, dirsize_quota_get_resource, dirsize_quota_update, + NULL, NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-fs.c --- a/src/plugins/quota/quota-fs.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-fs.c Tue Oct 16 03:34:51 2012 +0300 @@ -831,7 +831,8 @@ fs_quota_get_resource, fs_quota_update, - fs_quota_match_box + fs_quota_match_box, + NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-maildir.c --- a/src/plugins/quota/quota-maildir.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-maildir.c Tue Oct 16 03:34:51 2012 +0300 @@ -913,6 +913,7 @@ maildir_quota_root_get_resources, maildir_quota_get_resource, maildir_quota_update, + NULL, NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-private.h --- a/src/plugins/quota/quota-private.h Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-private.h Tue Oct 16 03:34:51 2012 +0300 @@ -67,7 +67,7 @@ int (*update)(struct quota_root *root, struct quota_transaction_context *ctx); bool (*match_box)(struct quota_root *root, struct mailbox *box); - + void (*flush)(struct quota_root *root); }; struct quota_backend { diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-storage.c Tue Oct 16 03:34:51 2012 +0300 @@ -366,14 +366,34 @@ return ret; } +static void quota_roots_flush(struct quota *quota) +{ + struct quota_root *const *roots; + unsigned int i, count; + + roots = array_get("a->roots, &count); + for (i = 0; i < count; i++) { + if (roots[i]->backend.v.flush != NULL) + roots[i]->backend.v.flush(roots[i]); + } +} + static void quota_mailbox_close(struct mailbox *box) { struct quota_mailbox *qbox = QUOTA_CONTEXT(box); + struct quota_user *quser = QUOTA_USER_CONTEXT(box->storage->user); /* sync_notify() may be called outside sync_begin()..sync_deinit(). make sure we apply changes at close time at latest. */ quota_mailbox_sync_commit(qbox); + /* make sure quota backend flushes all data. this could also be done + somewhat later, but user.deinit() is too late, since the flushing + can trigger quota recalculation which isn't safe to do anymore + at user.deinit() when most of the loaded plugins have already been + deinitialized. */ + quota_roots_flush(quser->quota); + qbox->module_ctx.super.close(box); } From pigeonhole at rename-it.nl Tue Oct 16 22:33:25 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 16 Oct 2012 21:33:25 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/12a4e11ecd4c changeset: 1660:12a4e11ecd4c user: Stephan Bosch date: Tue Oct 16 21:33:15 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Added new features to the duplicate test. It is now possible to track duplicates based on arbitrary headers or even arbitrary string values using the new :header and :value arguments respectively. The experation time can be configured using the new :seconds argument. This change is backwards compatible as long as the name argument wasn't used. This is now a :handle argument. diffstat: doc/rfc/spec-bosch-sieve-duplicate.txt | 284 +++++++-- doc/rfc/xml/reference.IMAP4FLAGS.xml | 15 + doc/rfc/xml/reference.MAILBOX.xml | 15 + doc/rfc/xml/reference.VACATION.xml | 17 + doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 164 ++++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 137 +++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 11 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c | 3 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 273 ++++++++- tests/extensions/vnd.dovecot/duplicate/errors.svtest | 2 +- tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve | 5 +- tests/extensions/vnd.dovecot/duplicate/execute.svtest | 10 +- 12 files changed, 747 insertions(+), 189 deletions(-) diffs (truncated from 1305 to 300 lines): diff -r 2b5ff3818a9f -r 12a4e11ecd4c doc/rfc/spec-bosch-sieve-duplicate.txt --- a/doc/rfc/spec-bosch-sieve-duplicate.txt Sat Oct 13 10:30:57 2012 +0200 +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt Tue Oct 16 21:33:15 2012 +0200 @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - February 25, 2012 + October 16, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -10,23 +10,65 @@ Abstract This document defines a new vendor-defined test command "duplicate" - for the "Sieve" email filtering language that tests whether an e-mail - message is a duplicate, i.e. whether it was seen before by the - delivery agent. Users can use this new test to remove duplicate - deliveries commonly caused by mailing list subscriptions or mail - account aliases. + for the "Sieve" email filtering language. It can be used to test + whether a particular string value is a duplicate, i.e. whether it was + seen before by the delivery agent that is executing the Sieve script. + The main application for this new test is detecting duplicate message + deliveries commonly caused by mailing list subscriptions or + redirected mail addresses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch [Page 1] + + Sieve: Detecting Duplicate Deliveries October 2012 Table of Contents - 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 2 - 2. Conventions Used in This Document . . . . . . . . . . . . . . . 2 - 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 2 - 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 3 - 5. Example . . . . . . . . . . . . . . . . . . . . . . . . . . . . 3 - 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 3 - 7. Normative References . . . . . . . . . . . . . . . . . . . . . 3 - Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . . 3 + 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 4 + 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 6 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 6 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 6 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 @@ -52,26 +94,52 @@ -Bosch [Page 1] + + + + + + + + + + + + + + +Bosch [Page 2] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 1. Introduction This is an extension to the Sieve filtering language defined by RFC - 5228 [SIEVE]. It adds a test to determine whether a message was seen - before by the delivery agent based on the Message-ID header. + 5228 [SIEVE]. It adds a test to determine whether a certain string + value was seen before by the delivery agent in an earlier execution + of the Sieve script. This can be used to detect and handle duplicate + message deliveries. Duplicate deliveries are a common side-effect of being subscribed to - a mailing list. If a member of the list decides to reply to both the - user and the mailing list itself, the user will get a copy of the - message directly and through mailing list. In another scenario, the - user has several aliases for his mail account and one of his contacts - sends the message to multiple addresses that eventually map to the - same account. Using the vnd.dovecot.duplicate extension, users have - the means to detect such duplicates and deal with these - appropriately, e.g. by discarding them. + a mailing list. For example, if a member of the list decides to + reply to both the user and the mailing list itself, the user will get + a copy of the message directly and through mailing list. Also, if + someone cross-posts over several mailing lists to which the user is + subscribed, the user will receive a copy from each of those lists. + In another scenario, the user has several redirected mail addresses + all pointing to his main mail account. If one of the user's contacts + sends the message to more than one of those addresses, the user will + receive more than a single copy. Using the "vnd.dovecot.duplicate" + extension, users have the means to detect and handle such duplicates, + e.g. by discarding them or putting them in a special folder. + + Duplicate messages are normally detected using the Message-ID header + field, which is required to be unique for each message. However, the + "duplicate" test is flexible enough to use different (weaker) + criteria for defining what makes a message a duplicate, for example + based on the subject line. Also, other applications of this new test + command are possible, as long as the tracked value is a string. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be @@ -89,31 +157,79 @@ arguments syntax. + + + + + + + +Bosch [Page 3] + + Sieve: Detecting Duplicate Deliveries October 2012 + + 3. Test "duplicate" - Usage: "duplicate" [] + Usage: "duplicate" [":seconds" ] + [":header" / + ":value" ] + [":handle" ] - The "duplicate" test keeps track of which Message-ID values were seen - before by this test in an earlier delivery operation. It evaluates - to "true" when the Message-ID header of the current message was seen - before. If it is not known, the test evaluates to "false" and the - Message-ID is added to a persistent internal tracking list. - Implementations SHOULD limit the number of messages that are tracked - and SHOULD let Message-ID entries expire after some short period of - time. + The "duplicate" test keeps track of which values were seen before by + this test in an earlier execution of this Sieve script. In its basic + form, the tested value is the content of the Message-ID header of the + message. This way, this test can be used to detect duplicate + deliveries of the same message. It can also detect duplicate + deliveries based on other message header fields if requested and it + can even use a user-provided string value, e.g. as composed from text + extracted from the message using the "variables" [VARIABLES] + extension. - Using the "name" argument, the duplicate test can be employed for - multiple independent purposes. Only when the Message-ID was seen - before in an earlier script execution by a duplicate test with the + The "duplicate" test evaluates to "true" when the provided value was + seen before. If the value is not known, the test evaluates to + "false" and the value is added to an internal value tracking list. + Implementations SHOULD limit the number of values (and thereby + messages) that are tracked. Also, implementations SHOULD let entries + in the value tracking list expire after a short period of time. + The user can explicitly control the length this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is + omitted, an appropriate default MUST be used. Sites SHOULD impose a + maximum limit on the expiration time. If that limit is exceeded, the + maximum value MUST silently be substituted; exceeding the limit MUST + NOT produce an error. + By default the tracked value is the content of the message's + Message-ID header field. For more advanced purposes, the content of + another header can be chosen for tracking by specifying the ":header" + argument. The tracked string value can also be specified explicitly + using the ":value" argument. The ":header" and ":value" arguments + are mutually exclusive and specifying both for a single "duplicate" + test command MUST trigger an error at compile time. If the value is + extracted from a header, i.e. when the ":value" argument is not used, + leading and trailing whitespace (see Section 2.2 of RFC 5228 [SIEVE]) + MUST first be trimmed from the value before executing the test. -Bosch [Page 2] + Using the ":handle" argument, the duplicate test can be employed for + multiple independent purposes. Only when the tracked value was seen + before in an earlier script execution by a "duplicate" test with the + same ":handle" argument, it is recognized as a duplicate. + + NOTE: The necessary mechanism to track duplicate messages is very + + + +Bosch [Page 4] - Sieve: Detecting Duplicate Deliveries February 2012 + Sieve: Detecting Duplicate Deliveries October 2012 - same "name" argument, it is recognized as a duplicate. + similar to the mechanism that is needed for tracking duplicate + responses for the "vacation" [VACATION] action. One way to implement + the necessary mechanism for the "duplicate" test is therefore to + store a hash of the tracked value and, if provided, the ":handle" + argument. 4. Sieve Capability Strings @@ -122,29 +238,71 @@ advertise the capability string "vnd.dovecot.duplicate". -5. Example +5. Examples - In this example, duplicate deliveries are stored in a special folder - contained in the user's Trash folder. If the folder does not exist, - it is created. This way, the user has a chance to recover messages - when necessary. Messages that are not recognized as duplicates are - stored in the user's inbox as normal. + In the following basic example, message duplicates are detected by + tracking the Message-ID header. Duplicate deliveries are stored in a + special folder contained in the user's Trash folder. If the folder + does not exist, it is created automatically using the "mailbox" + [MAILBOX] extension. This way, the user has a chance to recover + messages when necessary. Messages that are not recognized as + duplicates are stored in the user's inbox as normal. require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { - fileinto :create "Trash/Duplicate"; + fileinto :create "Trash/Duplicate"; } + The next example shows a more complex use of the "duplicate" test. + The user gets network alerts from a set of remote automated + monitoring systems. Multiple notifications can be received about the + same event from different monitoring systems. The Message-ID of + these messages is different, because these are all distinct messages + from different senders. To avoid being notified multiple times about + the same event the user writes the following script: + + require ["vnd.dovecot.duplicate", "variables", "imap4flags", + "fileinto"]; + + if header :matches "subject" "ALERT: *" { + if duplicate :seconds 60 :value "${1}" { + setflag "\\seen"; + } + fileinto "Alerts"; + } + + The subjects of the notification message are structured with a + + + From pigeonhole at rename-it.nl Wed Oct 17 03:15:09 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 02:15:09 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/3a972efe012b changeset: 1662:3a972efe012b user: Stephan Bosch date: Wed Oct 17 02:15:04 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Fixed segfault occurring in testsuite. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 2b767751ff27 -r 3a972efe012b src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:09:55 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:15:04 2012 +0200 @@ -334,7 +334,7 @@ } else if (value != NULL) { val = str_c(value); val_len = str_len(value); - } else { + } else if (renv->msgdata->id != NULL) { val = renv->msgdata->id; val_len = strlen(renv->msgdata->id); } From pigeonhole at rename-it.nl Wed Oct 17 03:15:09 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 02:15:09 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/2b767751ff27 changeset: 1661:2b767751ff27 user: Stephan Bosch date: Wed Oct 17 02:09:55 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: fixed bug in previous change. Forgot to initialize variable, causing a segfault at runtime. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 12a4e11ecd4c -r 2b767751ff27 src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Tue Oct 16 21:33:15 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:09:55 2012 +0200 @@ -277,7 +277,7 @@ const struct ext_duplicate_config *config = (const struct ext_duplicate_config *) ext->context; int opt_code = 0; - string_t *handle = NULL, *header = NULL, *value; + string_t *handle = NULL, *header = NULL, *value = NULL; const char *val = NULL; size_t val_len = 0; sieve_number_t seconds = config->default_period; From pigeonhole at rename-it.nl Wed Oct 17 03:59:33 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 02:59:33 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: vnd.dovecot.duplicate extensi... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/762b6a885897 changeset: 1663:762b6a885897 user: Stephan Bosch date: Wed Oct 17 02:55:07 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate extension: Only track duplicate when Sieve script executes successfully. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 111 ++++++++- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 2 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 4 +- 3 files changed, 96 insertions(+), 21 deletions(-) diffs (221 lines): diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 02:55:07 2012 +0200 @@ -5,6 +5,7 @@ #include "md5.h" #include "ioloop.h" #include "str.h" +#include "str-sanitize.h" #include "array.h" #include "sieve-common.h" @@ -15,6 +16,7 @@ #include "sieve-code.h" #include "sieve-runtime.h" #include "sieve-actions.h" +#include "sieve-result.h" #include "ext-duplicate-common.h" @@ -62,6 +64,67 @@ } /* + * Duplicate_mark action + */ + +struct act_duplicate_mark_data { + const char *handle; + unsigned int period; + unsigned char hash[MD5_RESULTLEN]; +}; + +static void act_duplicate_mark_print + (const struct sieve_action *action, + const struct sieve_result_print_env *rpenv, bool *keep); +static bool act_duplicate_mark_commit + (const struct sieve_action *action, + const struct sieve_action_exec_env *aenv, void *tr_context, bool *keep); + +static const struct sieve_action_def act_duplicate_mark = { + "duplicate_mark", + 0, + NULL, NULL, NULL, + act_duplicate_mark_print, + NULL, NULL, + act_duplicate_mark_commit, + NULL +}; + +static void act_duplicate_mark_print +(const struct sieve_action *action, + const struct sieve_result_print_env *rpenv, bool *keep ATTR_UNUSED) +{ + struct act_duplicate_mark_data *data = + (struct act_duplicate_mark_data *) action->context; + + if (data->handle != NULL) { + sieve_result_action_printf(rpenv, "track duplicate with handle: %s", + str_sanitize(data->handle, 128)); + } else { + sieve_result_action_printf(rpenv, "track duplicate"); + } +} + +static bool act_duplicate_mark_commit +(const struct sieve_action *action, + const struct sieve_action_exec_env *aenv, + void *tr_context ATTR_UNUSED, bool *keep ATTR_UNUSED) +{ + const struct sieve_script_env *senv = aenv->scriptenv; + struct act_duplicate_mark_data *data = + (struct act_duplicate_mark_data *) action->context; + + /* Message was handled successfully until now, so track duplicate for this + * message. + */ + sieve_action_duplicate_mark + (senv, data->hash, sizeof(data->hash), ioloop_time + data->period); + + return TRUE; +} + + +/* * Duplicate checking */ @@ -77,7 +140,7 @@ unsigned int nohandle_checked:1; }; -bool ext_duplicate_check +int ext_duplicate_check (const struct sieve_runtime_env *renv, string_t *handle, const char *value, size_t value_len, sieve_number_t period) { @@ -85,13 +148,13 @@ const struct sieve_script_env *senv = renv->scriptenv; struct ext_duplicate_context *rctx; bool duplicate = FALSE; - pool_t pool = NULL; + pool_t msg_pool = NULL, result_pool = NULL; static const char *id = "sieve duplicate"; - unsigned char dupl_hash[MD5_RESULTLEN]; + struct act_duplicate_mark_data *act; struct md5_context ctx; if ( !sieve_action_duplicate_check_available(senv) || value == NULL ) - return FALSE; + return 0; /* Get context; find out whether duplicate was checked earlier */ rctx = (struct ext_duplicate_context *) @@ -99,24 +162,30 @@ if ( rctx == NULL ) { /* Create context */ - pool = sieve_message_context_pool(renv->msgctx); - rctx = p_new(pool, struct ext_duplicate_context, 1); + msg_pool = sieve_message_context_pool(renv->msgctx); + rctx = p_new(msg_pool, struct ext_duplicate_context, 1); sieve_message_context_extension_set(renv->msgctx, this_ext, (void *)rctx); } else { if ( handle == NULL ) { if ( rctx->nohandle_checked ) { /* Already checked for duplicate */ - return rctx->nohandle_duplicate; + return ( rctx->nohandle_duplicate ? 1 : 0 ); } } else if ( array_is_created(&rctx->handles) ) { const struct ext_duplicate_handle *record; array_foreach (&rctx->handles, record) { if ( strcmp(record->handle, str_c(handle)) == 0 ) - return record->duplicate; + return ( record->duplicate ? 1 : 0 ); } } } + result_pool = sieve_result_pool(renv->result); + act = p_new(result_pool, struct act_duplicate_mark_data, 1); + if (handle != NULL) + act->handle = p_strdup(result_pool, str_c(handle)); + act->period = period; + /* Create hash */ md5_init(&ctx); md5_update(&ctx, id, strlen(id)); @@ -127,31 +196,35 @@ md5_update(&ctx, "default", 7); } md5_update(&ctx, value, value_len); - md5_final(&ctx, dupl_hash); + md5_final(&ctx, act->hash); /* Check duplicate */ - duplicate = sieve_action_duplicate_check - (senv, dupl_hash, sizeof(dupl_hash)); + duplicate = sieve_action_duplicate_check(senv, act->hash, sizeof(act->hash)); - /* Create/refresh entry */ - sieve_action_duplicate_mark - (senv, dupl_hash, sizeof(dupl_hash), ioloop_time + period); + /* We may only mark the message as duplicate when Sieve script executes + * successfully; therefore defer this operation until successful result + * execution. + */ + if ( sieve_result_add_action + (renv, NULL, &act_duplicate_mark, NULL, (void *) act, 0, FALSE) < 0 ) + return -1; + /* Cache result */ if ( handle == NULL ) { rctx->nohandle_duplicate = duplicate; rctx->nohandle_checked = TRUE; } else { struct ext_duplicate_handle *record; - if ( pool == NULL ) - pool = sieve_message_context_pool(renv->msgctx); + if ( msg_pool == NULL ) + msg_pool = sieve_message_context_pool(renv->msgctx); if ( !array_is_created(&rctx->handles) ) - p_array_init(&rctx->handles, pool, 64); + p_array_init(&rctx->handles, msg_pool, 64); record = array_append_space(&rctx->handles); - record->handle = p_strdup(pool, str_c(handle)); + record->handle = p_strdup(msg_pool, str_c(handle)); record->duplicate = duplicate; } - return duplicate; + return ( duplicate ? 1 : 0 ); } diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h Wed Oct 17 02:55:07 2012 +0200 @@ -38,7 +38,7 @@ * Duplicate checking */ -bool ext_duplicate_check +int ext_duplicate_check (const struct sieve_runtime_env *renv, string_t *handle, const char *value, size_t value_len, sieve_number_t period); diff -r 3a972efe012b -r 762b6a885897 src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:15:04 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c Wed Oct 17 02:55:07 2012 +0200 @@ -343,7 +343,9 @@ if (val == NULL) { duplicate = FALSE; } else { - duplicate = ext_duplicate_check(renv, handle, val, val_len, seconds); + if ((ret=ext_duplicate_check(renv, handle, val, val_len, seconds)) < 0) + return SIEVE_EXEC_FAILURE; + duplicate = ( ret > 0 ); } /* Trace */ From pigeonhole at rename-it.nl Wed Oct 17 22:50:00 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 21:50:00 +0200 Subject: dovecot-2.1-pigeonhole: Improved specification of the vnd.doveco... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/fd7257cb40ea changeset: 1664:fd7257cb40ea user: Stephan Bosch date: Wed Oct 17 21:49:55 2012 +0200 description: Improved specification of the vnd.dovecot.duplicate extension. diffstat: doc/rfc/spec-bosch-sieve-duplicate.txt | 132 ++++++++++++++-------------- doc/rfc/xml/reference.INCLUDE.xml | 17 +++ doc/rfc/xml/spec-bosch-sieve-duplicate.xml | 49 +++++++--- 3 files changed, 118 insertions(+), 80 deletions(-) diffs (truncated from 324 to 300 lines): diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/spec-bosch-sieve-duplicate.txt --- a/doc/rfc/spec-bosch-sieve-duplicate.txt Wed Oct 17 02:55:07 2012 +0200 +++ b/doc/rfc/spec-bosch-sieve-duplicate.txt Wed Oct 17 21:49:55 2012 +0200 @@ -2,7 +2,7 @@ Pigeonhole Project S. Bosch - October 16, 2012 + October 17, 2012 Sieve Email Filtering: Detecting Duplicate Deliveries @@ -65,9 +65,9 @@ 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 5 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 5 6. Security Considerations . . . . . . . . . . . . . . . . . . . . 6 - 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 6 - 7.1. Normative References . . . . . . . . . . . . . . . . . . . 6 - 7.2. Informative References . . . . . . . . . . . . . . . . . . 6 + 7. References . . . . . . . . . . . . . . . . . . . . . . . . . . 7 + 7.1. Normative References . . . . . . . . . . . . . . . . . . . 7 + 7.2. Informative References . . . . . . . . . . . . . . . . . . 7 Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 7 @@ -187,20 +187,50 @@ extension. The "duplicate" test evaluates to "true" when the provided value was - seen before. If the value is not known, the test evaluates to - "false" and the value is added to an internal value tracking list. + seen before in an earlier Sieve execution for a previous message + delivery. If the value was not seen earlier, the test evaluates to + "false". + + As a side-effect, the "duplicate" test adds the evaluated value to an + internal duplicate tracking list, so that the test will evaluate to + "true" the next time the Sieve script is executed and the same value + is encountered. Note that the "duplicate" test MUST only check for + duplicates amongst values encountered in previous executions of the + Sieve script; it MUST NOT consider values encountered earlier in the + current Sieve script execution as potential duplicates. This means + that all "duplicate" tests in a Sieve script execution, including + those located in scripts included using the "include" [INCLUDE] + extension, MUST yield the same result if the arguments are identical. + + Implementations MUST prevent adding values to the internal duplicate + tracking list when the Sieve script execution fails. For example, + this can be implemented by deferring the definitive modification of + the tracking list to the end of the Sieve script execution. If + failed script executions would add values to the duplicate tracking + list, all "duplicate" tests would erroneously yield "true" for the + next delivery attempt of the same message, which can -- depending on + the action taken for a duplicate -- easily lead to discarding the + message without further notice. + Implementations SHOULD limit the number of values (and thereby messages) that are tracked. Also, implementations SHOULD let entries - in the value tracking list expire after a short period of time. + in the value tracking list expire after a short period of time. The + user can explicitly control the length of this expiration time by + means of the ":seconds" argument. If the ":seconds" argument is - The user can explicitly control the length this expiration time by - means of the ":seconds" argument. If the ":seconds" argument is + + +Bosch [Page 4] + + Sieve: Detecting Duplicate Deliveries October 2012 + + omitted, an appropriate default MUST be used. Sites SHOULD impose a maximum limit on the expiration time. If that limit is exceeded, the maximum value MUST silently be substituted; exceeding the limit MUST NOT produce an error. - By default the tracked value is the content of the message's + By default, the tracked value is the content of the message's Message-ID header field. For more advanced purposes, the content of another header can be chosen for tracking by specifying the ":header" argument. The tracked string value can also be specified explicitly @@ -209,7 +239,8 @@ test command MUST trigger an error at compile time. If the value is extracted from a header, i.e. when the ":value" argument is not used, leading and trailing whitespace (see Section 2.2 of RFC 5228 [SIEVE]) - MUST first be trimmed from the value before executing the test. + MUST first be trimmed from the value before performing the actual + duplicate verification. Using the ":handle" argument, the duplicate test can be employed for multiple independent purposes. Only when the tracked value was seen @@ -217,14 +248,6 @@ same ":handle" argument, it is recognized as a duplicate. NOTE: The necessary mechanism to track duplicate messages is very - - - -Bosch [Page 4] - - Sieve: Detecting Duplicate Deliveries October 2012 - - similar to the mechanism that is needed for tracking duplicate responses for the "vacation" [VACATION] action. One way to implement the necessary mechanism for the "duplicate" test is therefore to @@ -248,6 +271,16 @@ messages when necessary. Messages that are not recognized as duplicates are stored in the user's inbox as normal. + + + + + +Bosch [Page 5] + + Sieve: Detecting Duplicate Deliveries October 2012 + + require ["vnd.dovecot.duplicate", "fileinto", "mailbox"]; if duplicate { @@ -273,14 +306,6 @@ } The subjects of the notification message are structured with a - - - -Bosch [Page 5] - - Sieve: Detecting Duplicate Deliveries October 2012 - - predictable pattern which includes a description of the event. In the script above the "duplicate" test is used to detect duplicate alert events. The message subject is matched against a pattern and @@ -290,7 +315,7 @@ due to the specified ":seconds" argument. In the the event of a duplicate, the message is marked as seen using the "imap4flags" [IMAP4FLAGS] extension. All alert messages are put into the "Alerts" - mailbox irrespective of wether those messages are duplicates or not. + mailbox irrespective of whether those messages are duplicates or not. 6. Security Considerations @@ -302,8 +327,21 @@ 7. References + + + + + +Bosch [Page 6] + + Sieve: Detecting Duplicate Deliveries October 2012 + + 7.1. Normative References + [INCLUDE] Daboo, C. and A. Stone, "Sieve Email Filtering: Include + Extension", RFC 6609, May 2012. + [KEYWORDS] Bradner, S., "Key words for use in RFCs to Indicate Requirement Levels", BCP 14, RFC 2119, March 1997. @@ -330,13 +368,6 @@ RFC 5229, January 2008. - - -Bosch [Page 6] - - Sieve: Detecting Duplicate Deliveries October 2012 - - Author's Address Stephan Bosch @@ -357,36 +388,5 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Bosch [Page 7] diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/xml/reference.INCLUDE.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/xml/reference.INCLUDE.xml Wed Oct 17 21:49:55 2012 +0200 @@ -0,0 +1,17 @@ + + + + + +Sieve Email Filtering: Include Extension + + + + + + +The Sieve Email Filtering "include" extension permits users to include one Sieve script inside another. This can make managing large scripts or multiple sets of scripts much easier, and allows a site and its users to build up libraries of scripts. Users are able to include their own personal scripts or site-wide scripts. [STANDARDS-TRACK] + + + + diff -r 762b6a885897 -r fd7257cb40ea doc/rfc/xml/spec-bosch-sieve-duplicate.xml --- a/doc/rfc/xml/spec-bosch-sieve-duplicate.xml Wed Oct 17 02:55:07 2012 +0200 +++ b/doc/rfc/xml/spec-bosch-sieve-duplicate.xml Wed Oct 17 21:49:55 2012 +0200 @@ -113,28 +113,48 @@ extension. The "duplicate" test evaluates to "true" when the provided value was seen -before. If the value is not known, the test evaluates to "false" and the value -is added to an internal value tracking list. Implementations SHOULD limit the -number of values (and thereby messages) that are tracked. Also, implementations -SHOULD let entries in the value tracking list expire after a short period of -time. +before in an earlier Sieve execution for a previous message delivery. If the +value was not seen earlier, the test evaluates to "false". -The user can explicitly control the length this expiration time by means of -the ":seconds" argument. If the ":seconds" argument is omitted, an appropriate -default MUST be used. Sites SHOULD impose a maximum limit on the expiration -time. If that limit is exceeded, the maximum value MUST silently be substituted; -exceeding the limit MUST NOT produce an error. +As a side-effect, the "duplicate" test adds the evaluated value to an +internal duplicate tracking list, so that the test will evaluate to "true" the +next time the Sieve script is executed and the same value is encountered. +Note that the "duplicate" test MUST only check for duplicates amongst values +encountered in previous executions of the Sieve script; it MUST NOT consider +values encountered earlier in the current Sieve script execution as potential +duplicates. This means that all "duplicate" tests in a Sieve script execution, +including those located in scripts included using the "include" + extension, MUST yield the same result if the arguments +are identical. -By default the tracked value is the content of the message's Message-ID +Implementations MUST prevent adding values to the internal duplicate tracking +list when the Sieve script execution fails. For example, this can be implemented +by deferring the definitive modification of the tracking list to the end of the +Sieve script execution. If failed script executions would add values to the +duplicate tracking list, all "duplicate" tests would erroneously yield "true" +for the next delivery attempt of the same message, which can -- depending on the +action taken for a duplicate -- easily lead to discarding the message without +further notice. + +Implementations SHOULD limit the number of values (and thereby messages) that +are tracked. Also, implementations SHOULD let entries in the value tracking list +expire after a short period of time. The user can explicitly control the length +of this expiration time by means of the ":seconds" argument. If the ":seconds" +argument is omitted, an appropriate default MUST be used. Sites SHOULD impose a +maximum limit on the expiration time. If that limit is exceeded, the maximum +value MUST silently be substituted; exceeding the limit MUST NOT produce an +error. + +By default, the tracked value is the content of the message's Message-ID header field. For more advanced purposes, the content of another header can be chosen for tracking by specifying the ":header" argument. The tracked string -value can also be specified explicitly using the ":value" argument. The +value can also be specified explicitly using the ":value" argument. The ":header" and ":value" arguments are mutually exclusive and specifying both for a single "duplicate" test command MUST trigger an error at compile time. If the value is extracted from a header, i.e. when the ":value" argument is not used, From pigeonhole at rename-it.nl Wed Oct 17 23:11:20 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 22:11:20 +0200 Subject: dovecot-2.1-pigeonhole: Updated installation documentation for v... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/0dc8453e07c4 changeset: 1666:0dc8453e07c4 user: Stephan Bosch date: Wed Oct 17 22:11:16 2012 +0200 description: Updated installation documentation for vnd.dovecot.duplicate extension. diffstat: doc/extensions/vnd.dovecot.duplicate.txt | 28 +++++++++++++++------------- 1 files changed, 15 insertions(+), 13 deletions(-) diffs (53 lines): diff -r c3b14a8784dc -r 0dc8453e07c4 doc/extensions/vnd.dovecot.duplicate.txt --- a/doc/extensions/vnd.dovecot.duplicate.txt Wed Oct 17 22:10:50 2012 +0200 +++ b/doc/extensions/vnd.dovecot.duplicate.txt Wed Oct 17 22:11:16 2012 +0200 @@ -8,13 +8,11 @@ Description =========== -Sieve (RFC 5228) is a highly extensible machine language specifically tailored -for internet message filtering. For the Dovecot Secure IMAP server, Sieve -support is provided by the Pigeonhole Sieve plugin. The vnd.dovecot.duplicate -extension augments the Sieve filtering implementation with a test to verify -whether a message was received earlier already based on its Message-ID. This can -be used to prevent duplicate deliveries, e.g. caused by mailinglists when people -reply both to the mailinglist and the user directly. +The vnd.dovecot.duplicate extension augments the Sieve filtering implementation +with a test to verify whether the evaluated string value was seen before in an +earlier execution of the Sieve script. The main application for this new test is +detecting and handling duplicate message deliveries, e.g. as caused by +mailinglists when people reply both to the mailinglist and the user directly. This extension is specific to the Pigeonhole Sieve implementation for the Dovecot Secure IMAP server. It will therefore most likely not be supported by @@ -35,14 +33,17 @@ ============= The "vnd.dovecot.duplicate" extension is not enabled by default and thus it -needs to be enabled explicitly. +needs to be enabled explicitly by adding it to the `sieve_extensions' or the +`sieve_global_extensions' setting. The following configuration settings are used: -sieve_duplicate_period = 1d - This option specifies after what period of time Message-IDs are purged from - the duplicate database. The period is specified in s(econds), unless followed - by a d(ay), h(our) or m(inute) specifier character. +sieve_duplicate_default_period = 12h +sieve_duplicate_max_period = 2d + These options respectively specify the default and the maximum value for the + period after which tracked values are purged from the duplicate tracking + database. The period is specified in s(econds), unless followed by a d(ay), + h(our) or m(inute) specifier character. Example ======= @@ -52,5 +53,6 @@ sieve_extensions = +vnd.dovecot.duplicate - sieve_duplicate_period = 6h + sieve_duplicate_default_period = 1h + sieve_duplicate_max_period = 1d } From pigeonhole at rename-it.nl Wed Oct 17 23:11:20 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 22:11:20 +0200 Subject: dovecot-2.1-pigeonhole: lib-sieve: vnd.dovecot.duplicate: Fixed ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.1-pigeonhole/rev/c3b14a8784dc changeset: 1665:c3b14a8784dc user: Stephan Bosch date: Wed Oct 17 22:10:50 2012 +0200 description: lib-sieve: vnd.dovecot.duplicate: Fixed default max period setting. diffstat: src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (22 lines): diff -r fd7257cb40ea -r c3b14a8784dc src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c --- a/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 21:49:55 2012 +0200 +++ b/src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c Wed Oct 17 22:10:50 2012 +0200 @@ -24,7 +24,8 @@ * Extension configuration */ -#define EXT_DUPLICATE_DEFAULT_PERIOD (1*24*60*60) +#define EXT_DUPLICATE_DEFAULT_PERIOD (12*60*60) +#define EXT_DUPLICATE_DEFAULT_MAX_PERIOD (2*24*60*60) bool ext_duplicate_load (const struct sieve_extension *ext, void **context) @@ -43,7 +44,7 @@ if ( !sieve_setting_get_duration_value (svinst, "sieve_duplicate_max_period", &max_period) ) { - max_period = EXT_DUPLICATE_DEFAULT_PERIOD; + max_period = EXT_DUPLICATE_DEFAULT_MAX_PERIOD; } config = i_new(struct ext_duplicate_config, 1); From pigeonhole at rename-it.nl Wed Oct 17 23:31:19 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 17 Oct 2012 22:31:19 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed a stupidity in the dict... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f147ddb83586 changeset: 1698:f147ddb83586 user: Stephan Bosch date: Wed Oct 17 22:22:02 2012 +0200 description: lib-sieve: Fixed a stupidity in the dict script implementation. diffstat: src/lib-sieve/sieve-script-dict.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6979dbc334fa -r f147ddb83586 src/lib-sieve/sieve-script-dict.c --- a/src/lib-sieve/sieve-script-dict.c Sat Oct 13 10:36:11 2012 +0200 +++ b/src/lib-sieve/sieve-script-dict.c Wed Oct 17 22:22:02 2012 +0200 @@ -121,7 +121,7 @@ if ( ret < 0 ) { sieve_critical(svinst, ehandler, name, "failed to open sieve script", "sieve dict backend: failed to initialize dict with data `%s' " - "for user `%s': error", data, username, error); + "for user `%s': %s", data, username, error); *error_r = SIEVE_ERROR_TEMP_FAIL; return -1; } From dovecot at dovecot.org Thu Oct 18 05:23:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 05:23:39 +0300 Subject: dovecot-2.1: lib-index: mail_cache_map() API cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/022d0d21e56d changeset: 14765:022d0d21e56d user: Timo Sirainen date: Thu Oct 18 05:10:29 2012 +0300 description: lib-index: mail_cache_map() API cleanup diffstat: src/lib-index/mail-cache-compress.c | 3 +- src/lib-index/mail-cache-fields.c | 46 +++++++++++++++++---------------- src/lib-index/mail-cache-lookup.c | 10 ++++-- src/lib-index/mail-cache-private.h | 3 +- src/lib-index/mail-cache-transaction.c | 17 +++++++----- src/lib-index/mail-cache.c | 36 ++++++++++++++++++++----- 6 files changed, 72 insertions(+), 43 deletions(-) diffs (truncated from 337 to 300 lines): diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:10:29 2012 +0300 @@ -353,6 +353,7 @@ uint32_t file_seq, old_offset; ARRAY_TYPE(uint32_t) ext_offsets; const uint32_t *offsets; + const void *data; unsigned int i, count; int fd, ret; @@ -440,7 +441,7 @@ if (cache->file_cache != NULL) file_cache_set_fd(cache->file_cache, cache->fd); - if (mail_cache_map(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) return -1; if (mail_cache_header_fields_read(cache) < 0) return -1; diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-fields.c Thu Oct 18 05:10:29 2012 +0300 @@ -198,11 +198,14 @@ return list; } -static int mail_cache_header_fields_get_offset(struct mail_cache *cache, - uint32_t *offset_r, bool map) +static int +mail_cache_header_fields_get_offset(struct mail_cache *cache, + uint32_t *offset_r, + const struct mail_cache_header_fields **field_hdr_r) { const struct mail_cache_header_fields *field_hdr; struct mail_cache_header_fields tmp_field_hdr; + const void *data; uint32_t offset = 0, next_offset; unsigned int next_count = 0; bool invalidate = FALSE; @@ -210,6 +213,8 @@ if (MAIL_CACHE_IS_UNUSABLE(cache)) { *offset_r = 0; + if (field_hdr_r != NULL) + *field_hdr_r = NULL; return 0; } @@ -228,16 +233,16 @@ invalidate = TRUE; if (cache->mmap_base != NULL) { - if (mail_cache_map(cache, offset, - sizeof(*field_hdr)) < 0) - return -1; - if (offset >= cache->mmap_length) { + ret = mail_cache_map(cache, offset, sizeof(*field_hdr), + &data); + if (ret <= 0) { + if (ret < 0) + return -1; mail_cache_set_corrupted(cache, "header field next_offset points outside file"); return -1; } - - field_hdr = CONST_PTR_OFFSET(cache->data, offset); + field_hdr = data; } else { /* if we need to follow multiple offsets to get to the last one, it's faster to just pread() the file @@ -270,7 +275,7 @@ if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT) cache->need_compress_file_seq = cache->hdr->file_seq; - if (map) { + if (field_hdr_r != NULL) { if (cache->file_cache != NULL && invalidate) { /* if this isn't the first header in file and we hadn't read this before, we can't trust that the cached @@ -278,17 +283,23 @@ file_cache_invalidate(cache->file_cache, offset, field_hdr->size); } - if (mail_cache_map(cache, offset, field_hdr->size) < 0) + ret = mail_cache_map(cache, offset, field_hdr->size, &data); + if (ret < 0) return -1; + if (ret == 0) { + mail_cache_set_corrupted(cache, + "header field size outside file"); + return -1; + } + *field_hdr_r = data; } - *offset_r = offset; return 0; } int mail_cache_header_fields_read(struct mail_cache *cache) { - const struct mail_cache_header_fields *field_hdr = NULL; + const struct mail_cache_header_fields *field_hdr; struct mail_cache_field field; const uint32_t *last_used, *sizes; const uint8_t *types, *decisions; @@ -299,7 +310,7 @@ time_t max_drop_time; uint32_t offset, i; - if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0) + if (mail_cache_header_fields_get_offset(cache, &offset, &field_hdr) < 0) return -1; if (offset == 0) { @@ -307,13 +318,6 @@ return 0; } - field_hdr = CONST_PTR_OFFSET(cache->data, offset); - if (offset + field_hdr->size > cache->mmap_length) { - mail_cache_set_corrupted(cache, - "field header points outside file"); - return -1; - } - /* check the fixed size of the header. name[] has to be checked separately */ if (field_hdr->size < sizeof(*field_hdr) + @@ -322,9 +326,7 @@ return -1; } - field_hdr = CONST_PTR_OFFSET(cache->data, offset); new_fields_count = field_hdr->fields_count; - if (new_fields_count != 0) { cache->file_field_map = i_realloc(cache->file_field_map, diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:10:29 2012 +0300 @@ -14,6 +14,7 @@ const struct mail_cache_record **rec_r) { const struct mail_cache_record *rec; + const void *data; i_assert(offset != 0); @@ -24,14 +25,15 @@ } /* we don't know yet how large the record is, so just guess */ - if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH) < 0) + if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH, + &data) < 0) return -1; if (offset + sizeof(*rec) > cache->mmap_length) { mail_cache_set_corrupted(cache, "record points outside file"); return -1; } - rec = CACHE_RECORD(cache, offset); + rec = data; if (rec->size < sizeof(*rec)) { mail_cache_set_corrupted(cache, "invalid record size"); @@ -39,9 +41,9 @@ } if (rec->size > CACHE_PREFETCH) { /* larger than we guessed. map the rest of the record. */ - if (mail_cache_map(cache, offset, rec->size) < 0) + if (mail_cache_map(cache, offset, rec->size, &data) < 0) return -1; - rec = CACHE_RECORD(cache, offset); + rec = data; } if (rec->size > cache->mmap_length || diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:10:29 2012 +0300 @@ -256,7 +256,8 @@ int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx, struct mail_cache_iterate_field *field_r); -int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size); +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r); void mail_cache_file_close(struct mail_cache *cache); int mail_cache_reopen(struct mail_cache *cache); diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-transaction.c Thu Oct 18 05:10:29 2012 +0300 @@ -1091,6 +1091,8 @@ uint32_t new_offset) { const struct mail_cache_record *rec; + const void *data; + int ret; i_assert(cache->locked); @@ -1107,15 +1109,16 @@ records at the same time. we'd rather not lose those additions, so force the linking order to be new_offset -> old_offset if it isn't already. */ - if (mail_cache_map(cache, new_offset, sizeof(*rec)) < 0) - return -1; - if (new_offset + sizeof(*rec) > cache->mmap_length) { - mail_cache_set_corrupted(cache, - "Cache record offset %u points outside file", - new_offset); + ret = mail_cache_map(cache, new_offset, sizeof(*rec), &data); + if (ret <= 0) { + if (ret == 0) { + mail_cache_set_corrupted(cache, + "Cache record offset %u points outside file", + new_offset); + } return -1; } - rec = CACHE_RECORD(cache, new_offset); + rec = data; if (rec->prev_offset == old_offset) { /* link is already correct */ return 0; diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:10:29 2012 +0300 @@ -145,6 +145,7 @@ { struct mail_index_view *view; const struct mail_index_ext *ext; + const void *data; i_assert(!cache->locked); @@ -167,7 +168,7 @@ mail_cache_init_file_cache(cache); - if (mail_cache_map(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) return -1; if (mail_cache_header_fields_read(cache) < 0) @@ -265,8 +266,10 @@ return TRUE; } -int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size) +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) { + const void *data; ssize_t ret; cache->remap_counter++; @@ -309,13 +312,22 @@ cache->hdr = &cache->hdr_ro_copy; if (offset == 0) mail_cache_update_need_compress(cache); - return 0; + + data = file_cache_get_map(cache->file_cache, + &cache->mmap_length); + if (offset > cache->mmap_length) { + *data_r = NULL; + return 0; + } + *data_r = CONST_PTR_OFFSET(data, offset); + return offset + size > cache->mmap_length ? 0 : 1; } if (offset < cache->mmap_length && size <= cache->mmap_length - offset) { /* already mapped */ - return 0; + *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); + return 1; } if (cache->mmap_base != NULL) { @@ -355,11 +367,18 @@ cache->hdr = cache->data; if (offset == 0) mail_cache_update_need_compress(cache); - return 0; + if (offset > cache->mmap_length) { + *data_r = NULL; + return 0; + } + *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); + return offset + size > cache->mmap_length ? 0 : 1; } From dovecot at dovecot.org Thu Oct 18 05:23:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 05:23:39 +0300 Subject: dovecot-2.1: lib-index: Code cleanup: Removed mail_cache.data Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0d1de37ad9d8 changeset: 14766:0d1de37ad9d8 user: Timo Sirainen date: Thu Oct 18 05:16:54 2012 +0300 description: lib-index: Code cleanup: Removed mail_cache.data diffstat: src/lib-index/mail-cache-lookup.c | 10 +++++++--- src/lib-index/mail-cache-private.h | 8 ++------ src/lib-index/mail-cache.c | 30 +++++++++--------------------- 3 files changed, 18 insertions(+), 30 deletions(-) diffs (176 lines): diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:16:54 2012 +0300 @@ -272,6 +272,7 @@ field_r->field_idx = field_idx; field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos); field_r->size = data_size; + field_r->offset = ctx->offset + ctx->pos; /* each record begins from 32bit aligned position */ ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1); @@ -446,8 +447,7 @@ lines_count = i; hdr_data = t_new(struct header_lookup_data, 1); - hdr_data->offset = (const char *)&lines[lines_count+1] - - (const char *)ctx->view->cache->data; + hdr_data->offset = field->offset + (lines_count+1) * sizeof(uint32_t); hdr_data->data_size = data_size; for (i = 0; i < lines_count; i++) { @@ -473,6 +473,7 @@ struct mail_cache_iterate_field field; struct header_lookup_context ctx; struct header_lookup_line *lines; + const void *data; const unsigned char *p, *start, *end; uint8_t *field_state; unsigned int i, count, max_field = 0; @@ -537,7 +538,10 @@ /* then start filling dest buffer from the headers */ for (i = 0; i < count; i++) { - start = CONST_PTR_OFFSET(cache->data, lines[i].data->offset); + if (mail_cache_map(cache, lines[i].data->offset, + lines[i].data->data_size, &data) <= 0) + return -1; + start = data; end = start + lines[i].data->data_size; /* find the end of the (multiline) header */ diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300 @@ -39,10 +39,6 @@ #define MAIL_CACHE_LOCK_TIMEOUT 10 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 -#define CACHE_RECORD(cache, offset) \ - ((const struct mail_cache_record *) \ - ((const char *) (cache)->data + offset)) - #define MAIL_CACHE_IS_UNUSABLE(cache) \ ((cache)->hdr == NULL) @@ -135,7 +131,6 @@ dev_t st_dev; void *mmap_base; - const void *data; size_t mmap_length; struct file_cache *file_cache; /* mail_cache_map() increases this always. */ @@ -203,8 +198,9 @@ struct mail_cache_iterate_field { unsigned int field_idx; + unsigned int size; const void *data; - unsigned int size; + uoff_t offset; }; struct mail_cache_lookup_iterate_ctx { diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300 @@ -59,7 +59,6 @@ file_cache_set_fd(cache->file_cache, -1); cache->mmap_base = NULL; - cache->data = NULL; cache->hdr = NULL; cache->mmap_length = 0; cache->last_field_header_offset = 0; @@ -214,10 +213,9 @@ cache->need_compress_file_seq = hdr->file_seq; } -static bool mail_cache_verify_header(struct mail_cache *cache) +static bool mail_cache_verify_header(struct mail_cache *cache, + const struct mail_cache_header *hdr) { - const struct mail_cache_header *hdr = cache->data; - /* check that the header is still ok */ if (cache->mmap_length < sizeof(struct mail_cache_header)) { mail_cache_set_corrupted(cache, "File too small"); @@ -278,7 +276,6 @@ size = sizeof(struct mail_cache_header); if (cache->file_cache != NULL) { - cache->data = NULL; cache->hdr = NULL; ret = file_cache_read(cache->file_cache, offset, size); @@ -295,26 +292,24 @@ return -1; } - cache->data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); + data = file_cache_get_map(cache->file_cache, + &cache->mmap_length); if (offset == 0) { - if (!mail_cache_verify_header(cache)) { + if (!mail_cache_verify_header(cache, data)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? cache->hdr->file_seq : 0; return -1; } - memcpy(&cache->hdr_ro_copy, cache->data, + memcpy(&cache->hdr_ro_copy, data, sizeof(cache->hdr_ro_copy)); } cache->hdr = &cache->hdr_ro_copy; if (offset == 0) mail_cache_update_need_compress(cache); - data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); if (offset > cache->mmap_length) { *data_r = NULL; return 0; @@ -350,13 +345,11 @@ cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length); if (cache->mmap_base == MAP_FAILED) { cache->mmap_base = NULL; - cache->data = NULL; mail_cache_set_syscall_error(cache, "mmap()"); return -1; } - cache->data = cache->mmap_base; - if (!mail_cache_verify_header(cache)) { + if (!mail_cache_verify_header(cache, cache->mmap_base)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? @@ -364,7 +357,7 @@ return -1; } - cache->hdr = cache->data; + cache->hdr = cache->mmap_base; if (offset == 0) mail_cache_update_need_compress(cache); if (offset > cache->mmap_length) { @@ -698,13 +691,8 @@ return -1; } - if (cache->file_cache != NULL) { + if (cache->file_cache != NULL) file_cache_write(cache->file_cache, data, size, offset); - - /* data pointer may change if file cache was grown */ - cache->data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); - } return 0; } From dovecot at dovecot.org Thu Oct 18 05:23:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 05:23:39 +0300 Subject: dovecot-2.1: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to d... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/51f2be9aa8ad changeset: 14767:51f2be9aa8ad user: Timo Sirainen date: Thu Oct 18 05:22:36 2012 +0300 description: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. diffstat: src/lib-index/mail-cache-compress.c | 4 + src/lib-index/mail-cache-private.h | 8 ++- src/lib-index/mail-cache.c | 123 +++++++++++++++++++++++------------ src/lib-index/mail-index.h | 5 +- 4 files changed, 96 insertions(+), 44 deletions(-) diffs (232 lines): diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:22:36 2012 +0300 @@ -461,6 +461,9 @@ if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) return 0; + /* compression isn't very efficient with small read()s */ + cache->map_with_read = FALSE; + if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates the dotlock file we need. */ @@ -495,5 +498,6 @@ bool mail_cache_need_compress(struct mail_cache *cache) { return cache->need_compress_file_seq != 0 && + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 && !cache->index->readonly; } diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:22:36 2012 +0300 @@ -130,9 +130,14 @@ ino_t st_ino; dev_t st_dev; + size_t mmap_length; + /* a) mmaping the whole file */ void *mmap_base; - size_t mmap_length; + /* b) using file cache */ struct file_cache *file_cache; + /* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */ + uoff_t read_offset; + buffer_t *read_buf; /* mail_cache_map() increases this always. */ unsigned int remap_counter; @@ -169,6 +174,7 @@ unsigned int hdr_modified:1; unsigned int field_header_write_pending:1; unsigned int compressing:1; + unsigned int map_with_read:1; }; struct mail_cache_loop_track { diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:22:36 2012 +0300 @@ -7,6 +7,7 @@ #include "nfs-workarounds.h" #include "file-cache.h" #include "mmap-util.h" +#include "read-full.h" #include "write-full.h" #include "mail-cache-private.h" @@ -264,17 +265,87 @@ return TRUE; } +static int +mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, + const void *data, bool copy_hdr) +{ + if (offset == 0) { + const struct mail_cache_header *hdr = data; + + if (!mail_cache_verify_header(cache, hdr)) { + cache->need_compress_file_seq = + !MAIL_CACHE_IS_UNUSABLE(cache) && + cache->hdr->file_seq != 0 ? + cache->hdr->file_seq : 0; + return -1; + } + if (!copy_hdr) + cache->hdr = data; + else { + memcpy(&cache->hdr_ro_copy, data, + sizeof(cache->hdr_ro_copy)); + cache->hdr = &cache->hdr_ro_copy; + } + mail_cache_update_need_compress(cache); + } + + if (offset + size > cache->mmap_length) + return 0; + return 1; +} + +static int +mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) +{ + void *data; + ssize_t ret; + + if (cache->read_buf == NULL) { + cache->read_buf = + buffer_create_dynamic(default_pool, size); + } else if (cache->read_offset <= offset && + cache->read_offset + cache->read_buf->used >= offset+size) { + /* already mapped */ + *data_r = CONST_PTR_OFFSET(cache->mmap_base, + offset - cache->read_offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + } else { + buffer_set_used_size(cache->read_buf, 0); + } + data = buffer_append_space_unsafe(cache->read_buf, size); + ret = pread(cache->fd, data, size, offset); + if (ret < 0) { + if (errno != ESTALE) + mail_cache_set_syscall_error(cache, "read()"); + + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + return -1; + } + buffer_set_used_size(cache->read_buf, ret); + + cache->read_offset = offset; + cache->mmap_length = offset + size; + + *data_r = data; + return mail_cache_map_finish(cache, offset, size, data, TRUE); +} + int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { const void *data; ssize_t ret; - cache->remap_counter++; - if (size == 0) size = sizeof(struct mail_cache_header); + cache->remap_counter++; + if (cache->map_with_read) + return mail_cache_map_with_read(cache, offset, size, data_r); + if (cache->file_cache != NULL) { cache->hdr = NULL; @@ -294,28 +365,9 @@ data = file_cache_get_map(cache->file_cache, &cache->mmap_length); - - if (offset == 0) { - if (!mail_cache_verify_header(cache, data)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - memcpy(&cache->hdr_ro_copy, data, - sizeof(cache->hdr_ro_copy)); - } - cache->hdr = &cache->hdr_ro_copy; - if (offset == 0) - mail_cache_update_need_compress(cache); - - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(data, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(data, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); } if (offset < cache->mmap_length && @@ -348,24 +400,9 @@ mail_cache_set_syscall_error(cache, "mmap()"); return -1; } - - if (!mail_cache_verify_header(cache, cache->mmap_base)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - - cache->hdr = cache->mmap_base; - if (offset == 0) - mail_cache_update_need_compress(cache); - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(cache->mmap_base, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, FALSE); } static int mail_cache_try_open(struct mail_cache *cache) @@ -437,6 +474,8 @@ if (!MAIL_INDEX_IS_IN_MEMORY(index) && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0) cache->file_cache = file_cache_new(-1); + cache->map_with_read = + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0; cache->ext_id = mail_index_ext_register(index, "cache", 0, diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-index.h Thu Oct 18 05:22:36 2012 +0300 @@ -27,7 +27,10 @@ MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100, /* If we run out of disk space, fail modifications instead of moving indexes to memory. */ - MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200 + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200, + /* We're only going to save new messages to the index. + Avoid unnecessary reads. */ + MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400 }; enum mail_index_header_compat_flags { From dovecot at dovecot.org Thu Oct 18 05:23:39 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 05:23:39 +0300 Subject: dovecot-2.1: lib-storage: Open index with MAIL_INDEX_OPEN_FLAG_S... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b7c6215b2b7c changeset: 14768:b7c6215b2b7c user: Timo Sirainen date: Thu Oct 18 05:23:27 2012 +0300 description: lib-storage: Open index with MAIL_INDEX_OPEN_FLAG_SAVEONLY if mailbox has MAILBOX_FLAG_SAVEONLY diffstat: src/lib-storage/index/index-storage.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 51f2be9aa8ad -r b7c6215b2b7c src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Thu Oct 18 05:22:36 2012 +0300 +++ b/src/lib-storage/index/index-storage.c Thu Oct 18 05:23:27 2012 +0300 @@ -304,6 +304,8 @@ ibox->list_index_sync_ext_id = (uint32_t)-1; ibox->index_flags = MAIL_INDEX_OPEN_FLAG_CREATE | mail_storage_settings_to_index_flags(box->storage->set); + if ((box->flags & MAILBOX_FLAG_SAVEONLY) != 0) + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_SAVEONLY; ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; MODULE_CONTEXT_SET(box, index_storage_module, ibox); From dovecot at dovecot.org Thu Oct 18 06:00:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:00:31 +0300 Subject: dovecot-2.1: lib-index: Handle better race condition there dovec... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/bcef97fb1202 changeset: 14769:bcef97fb1202 user: Timo Sirainen date: Thu Oct 18 05:55:30 2012 +0300 description: lib-index: Handle better race condition there dovecot.index.log and .log.2 are the same link. diffstat: src/lib-index/mail-transaction-log-file.c | 15 ++++++++++----- src/lib-index/mail-transaction-log-private.h | 3 +-- src/lib-index/mail-transaction-log.c | 6 +++--- 3 files changed, 14 insertions(+), 10 deletions(-) diffs (83 lines): diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:55:30 2012 +0300 @@ -756,6 +756,9 @@ second log file and we're going to overwrite this first one. */ } + /* NOTE: here's a race condition where both .log and .log.2 + point to the same file. our reading code should ignore that + though by comparing the inodes. */ } if (file_dotlock_replace(dotlock, @@ -809,8 +812,7 @@ return 0; } -int mail_transaction_log_file_open(struct mail_transaction_log_file *file, - bool check_existing) +int mail_transaction_log_file_open(struct mail_transaction_log_file *file) { struct mail_index *index = file->log->index; unsigned int i; @@ -834,10 +836,13 @@ ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT; if (mail_transaction_log_file_stat(file, ignore_estale) < 0) ret = -1; - else if (check_existing && - mail_transaction_log_file_is_dupe(file)) + else if (mail_transaction_log_file_is_dupe(file)) { + /* probably our already opened .log file has been + renamed to .log.2 and we're trying to reopen it. + also possible that hit a race condition where .log + and .log.2 are linked. */ return 0; - else { + } else { ret = mail_transaction_log_file_read_hdr(file, ignore_estale); } diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log-private.h --- a/src/lib-index/mail-transaction-log-private.h Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log-private.h Thu Oct 18 05:55:30 2012 +0300 @@ -117,8 +117,7 @@ const char *path); void mail_transaction_log_file_free(struct mail_transaction_log_file **file); -int mail_transaction_log_file_open(struct mail_transaction_log_file *file, - bool check_existing); +int mail_transaction_log_file_open(struct mail_transaction_log_file *file); int mail_transaction_log_file_create(struct mail_transaction_log_file *file, bool reset); int mail_transaction_log_file_lock(struct mail_transaction_log_file *file); diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log.c Thu Oct 18 05:55:30 2012 +0300 @@ -84,7 +84,7 @@ return 0; file = mail_transaction_log_file_alloc(log, log->filepath); - if ((ret = mail_transaction_log_file_open(file, FALSE)) <= 0) { + if ((ret = mail_transaction_log_file_open(file)) <= 0) { /* leave the file for _create() */ log->open_file = file; return ret; @@ -322,7 +322,7 @@ } file = mail_transaction_log_file_alloc(log, log->filepath); - if (mail_transaction_log_file_open(file, FALSE) <= 0) { + if (mail_transaction_log_file_open(file) <= 0) { mail_transaction_log_file_free(&file); return -1; } @@ -400,7 +400,7 @@ /* see if we have it in log.2 file */ file = mail_transaction_log_file_alloc(log, log->filepath2); - if ((ret = mail_transaction_log_file_open(file, TRUE)) <= 0) { + if ((ret = mail_transaction_log_file_open(file)) <= 0) { mail_transaction_log_file_free(&file); return ret; } From dovecot at dovecot.org Thu Oct 18 06:00:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:00:31 +0300 Subject: dovecot-2.1: lib-index: Fixed handling of finding a duplicate do... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/be50d12be960 changeset: 14770:be50d12be960 user: Timo Sirainen date: Thu Oct 18 06:00:18 2012 +0300 description: lib-index: Fixed handling of finding a duplicate dovecot.index.log file_seq Previously we assumed that the already opened file was always the wrong one, but more common was that the newly opened file was .log.2 which should have been deleted. diffstat: src/lib-index/mail-transaction-log-file.c | 47 +++++++++++++++++++----------- 1 files changed, 30 insertions(+), 17 deletions(-) diffs (67 lines): diff -r bcef97fb1202 -r be50d12be960 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:55:30 2012 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Thu Oct 18 06:00:18 2012 +0300 @@ -443,6 +443,32 @@ } static int +mail_transaction_log_file_fail_dupe(struct mail_transaction_log_file *file) +{ + int ret; + + /* mark the old file corrupted. we can't safely remove + it from the list however, so return failure. */ + file->hdr.indexid = 0; + if (strcmp(file->filepath, file->log->head->filepath) != 0) { + /* only mark .2 corrupted, just to make sure we don't lose any + changes from .log in case we're somehow wrong */ + mail_transaction_log_mark_corrupted(file); + ret = 0; + } else { + ret = -1; + } + if (!file->corrupted) { + file->corrupted = TRUE; + mail_index_set_error(file->log->index, + "Transaction log %s: " + "duplicate transaction log sequence (%u)", + file->filepath, file->hdr.file_seq); + } + return ret; +} + +static int mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file, bool ignore_estale) { @@ -528,26 +554,13 @@ corrupted. */ for (f = file->log->files; f != NULL; f = f->next) { if (f->hdr.file_seq == file->hdr.file_seq) { - /* mark the old file corrupted. we can't safely remove - it from the list however, so return failure. */ - f->hdr.indexid = 0; if (strcmp(f->filepath, f->log->head->filepath) != 0) { - /* only mark .2 corrupted, just to make sure - we don't lose any changes from .log in case - we're somehow wrong */ - mail_transaction_log_mark_corrupted(f); - ret = 0; + /* old "f" is the .log.2 */ + return mail_transaction_log_file_fail_dupe(f); } else { - ret = -1; + /* new "file" is probably the .log.2 */ + return mail_transaction_log_file_fail_dupe(file); } - if (!f->corrupted) { - f->corrupted = TRUE; - mail_index_set_error(f->log->index, - "Transaction log %s: " - "duplicate transaction log sequence (%u)", - f->filepath, f->hdr.file_seq); - } - return ret; } } From dovecot at dovecot.org Thu Oct 18 06:23:35 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:23:35 +0300 Subject: dovecot-2.1: login_log_format_elements: Added %{real_rip} variable. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/92364817f4ba changeset: 14771:92364817f4ba user: Timo Sirainen date: Thu Oct 18 06:21:25 2012 +0300 description: login_log_format_elements: Added %{real_rip} variable. It differs from %r when Dovecot proxy sends an updated client IP address. Patch by Jack Bates. diffstat: src/login-common/client-common.c | 3 +++ src/login-common/client-common.h | 1 + 2 files changed, 4 insertions(+), 0 deletions(-) diffs (38 lines): diff -r be50d12be960 -r 92364817f4ba src/login-common/client-common.c --- a/src/login-common/client-common.c Thu Oct 18 06:00:18 2012 +0300 +++ b/src/login-common/client-common.c Thu Oct 18 06:21:25 2012 +0300 @@ -97,6 +97,7 @@ client->set = set; client->local_ip = *local_ip; client->ip = *remote_ip; + client->real_ip = *remote_ip; client->fd = fd; client->tls = ssl; client->trusted = client_is_trusted(client); @@ -429,6 +430,7 @@ { 'k', NULL, "ssl_security" }, { 'e', NULL, "mail_pid" }, { '\0', NULL, "session" }, + { '\0', NULL, "real_rip" }, { '\0', NULL, NULL } }; @@ -478,6 +480,7 @@ tab[13].value = client->mail_pid == 0 ? "" : dec2str(client->mail_pid); tab[14].value = client_get_session_id(client); + tab[15].value = net_ip2addr(&client->real_ip); return tab; } diff -r be50d12be960 -r 92364817f4ba src/login-common/client-common.h --- a/src/login-common/client-common.h Thu Oct 18 06:00:18 2012 +0300 +++ b/src/login-common/client-common.h Thu Oct 18 06:21:25 2012 +0300 @@ -88,6 +88,7 @@ struct ip_addr local_ip; struct ip_addr ip; + struct ip_addr real_ip; unsigned int local_port, remote_port; struct ssl_proxy *ssl_proxy; const struct login_settings *set; From dovecot at dovecot.org Thu Oct 18 06:45:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:45:53 +0300 Subject: dovecot-2.1: lib-index: Crashfix for MAIL_INDEX_OPEN_FLAG_SAVEON... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/99aec8d4d6c6 changeset: 14772:99aec8d4d6c6 user: Timo Sirainen date: Thu Oct 18 06:45:39 2012 +0300 description: lib-index: Crashfix for MAIL_INDEX_OPEN_FLAG_SAVEONLY change. diffstat: src/lib-index/mail-cache.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 92364817f4ba -r 99aec8d4d6c6 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 06:21:25 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 06:45:39 2012 +0300 @@ -307,7 +307,7 @@ } else if (cache->read_offset <= offset && cache->read_offset + cache->read_buf->used >= offset+size) { /* already mapped */ - *data_r = CONST_PTR_OFFSET(cache->mmap_base, + *data_r = CONST_PTR_OFFSET(cache->read_buf->data, offset - cache->read_offset); return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); } else { From dovecot at dovecot.org Thu Oct 18 06:55:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:55:36 +0300 Subject: dovecot-2.2: Moved random_init() from lib-imap-urlauth to imap/i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/70ca88e74a4b changeset: 15227:70ca88e74a4b user: Timo Sirainen date: Thu Oct 18 06:55:04 2012 +0300 description: Moved random_init() from lib-imap-urlauth to imap/imap-urlauth main(). This way if the processes are chrooted they are still able to open /dev/urandom. diffstat: src/imap-urlauth/imap-urlauth-worker.c | 3 +++ src/imap/main.c | 3 +++ src/lib-imap-urlauth/imap-urlauth.c | 4 ---- 3 files changed, 6 insertions(+), 4 deletions(-) diffs (75 lines): diff -r 0bb400d84134 -r 70ca88e74a4b src/imap-urlauth/imap-urlauth-worker.c --- a/src/imap-urlauth/imap-urlauth-worker.c Sat Oct 13 04:30:17 2012 +0300 +++ b/src/imap-urlauth/imap-urlauth-worker.c Thu Oct 18 06:55:04 2012 +0300 @@ -14,6 +14,7 @@ #include "hostpid.h" #include "var-expand.h" #include "process-title.h" +#include "randgen.h" #include "restrict-access.h" #include "settings-parser.h" #include "master-service.h" @@ -1011,6 +1012,7 @@ master_service_init_finish(master_service); master_service_set_die_callback(master_service, imap_urlauth_worker_die); + random_init(); storage_service = mail_storage_service_init(master_service, set_roots, storage_service_flags); @@ -1037,6 +1039,7 @@ clients_destroy_all(); mail_storage_service_deinit(&storage_service); + random_deinit(); master_service_deinit(&master_service); return 0; } diff -r 0bb400d84134 -r 70ca88e74a4b src/imap/main.c --- a/src/imap/main.c Sat Oct 13 04:30:17 2012 +0300 +++ b/src/imap/main.c Thu Oct 18 06:55:04 2012 +0300 @@ -8,6 +8,7 @@ #include "str.h" #include "base64.h" #include "process-title.h" +#include "randgen.h" #include "restrict-access.h" #include "fd-close-on-exec.h" #include "settings-parser.h" @@ -369,6 +370,7 @@ commands_init(); imap_fetch_handlers_init(); + random_init(); storage_service = mail_storage_service_init(master_service, set_roots, storage_service_flags); @@ -405,6 +407,7 @@ imap_fetch_handlers_deinit(); commands_deinit(); + random_deinit(); master_service_deinit(&master_service); return 0; } diff -r 0bb400d84134 -r 70ca88e74a4b src/lib-imap-urlauth/imap-urlauth.c --- a/src/lib-imap-urlauth/imap-urlauth.c Sat Oct 13 04:30:17 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth.c Thu Oct 18 06:55:04 2012 +0300 @@ -37,8 +37,6 @@ i_assert(*config->url_host != '\0'); - random_init(); - uctx = i_new(struct imap_urlauth_context, 1); uctx->user = user; uctx->url_host = i_strdup(config->url_host); @@ -78,8 +76,6 @@ i_free(uctx->access_user); i_free(uctx->access_applications); i_free(uctx); - - random_deinit(); } static const unsigned char * From dovecot at dovecot.org Thu Oct 18 06:58:07 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Oct 2012 06:58:07 +0300 Subject: dovecot-2.2: lib-imap-urlauth: Minor fix to avoiding timing atta... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b21fe1a1c7ad changeset: 15228:b21fe1a1c7ad user: Timo Sirainen date: Thu Oct 18 06:58:01 2012 +0300 description: lib-imap-urlauth: Minor fix to avoiding timing attacks against mailbox existence. random_fill() is slightly slower than random_fill_weak(). Probably even better way to handle timing attacks would be to always add a short random pause before returning verification failure. diffstat: src/lib-imap-urlauth/imap-urlauth.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 70ca88e74a4b -r b21fe1a1c7ad src/lib-imap-urlauth/imap-urlauth.c --- a/src/lib-imap-urlauth/imap-urlauth.c Thu Oct 18 06:55:04 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth.c Thu Oct 18 06:58:01 2012 +0300 @@ -399,7 +399,7 @@ random "plausible" keys (selected by the server) as needed, before returning a validation failure. This prevents timing attacks aimed at identifying mailbox names.' */ - random_fill_weak(mailbox_key, sizeof(mailbox_key)); + random_fill(mailbox_key, sizeof(mailbox_key)); (void)imap_urlauth_internal_verify(url->uauth_rumpurl, mailbox_key, url->uauth_token, url->uauth_token_size); From dovecot at dovecot.org Sun Oct 21 07:14:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 21 Oct 2012 07:14:01 +0300 Subject: dovecot-2.1: redis dict: Log an error if we get disconnected une... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/a97c62fdd4ad changeset: 14773:a97c62fdd4ad user: Timo Sirainen date: Sun Oct 21 07:13:44 2012 +0300 description: redis dict: Log an error if we get disconnected unexpectedly. diffstat: src/lib-dict/dict-redis.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 99aec8d4d6c6 -r a97c62fdd4ad src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Thu Oct 18 06:45:39 2012 +0300 +++ b/src/lib-dict/dict-redis.c Sun Oct 21 07:13:44 2012 +0300 @@ -241,6 +241,8 @@ case 0: return; case -1: + if (conn->dict->ioloop != NULL) + i_error("redis: Disconnected unexpectedly"); redis_conn_destroy(_conn); return; default: From dovecot at dovecot.org Sun Oct 21 07:29:16 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 21 Oct 2012 07:29:16 +0300 Subject: dovecot-2.2: lib-fs: Fixes to async APIs. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/260b1ec12f41 changeset: 15229:260b1ec12f41 user: Timo Sirainen date: Sun Oct 21 07:29:04 2012 +0300 description: lib-fs: Fixes to async APIs. diffstat: src/lib-fs/fs-api.c | 10 ++++++++++ src/lib-fs/fs-api.h | 5 ++++- 2 files changed, 14 insertions(+), 1 deletions(-) diffs (50 lines): diff -r b21fe1a1c7ad -r 260b1ec12f41 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Oct 18 06:58:01 2012 +0300 +++ b/src/lib-fs/fs-api.c Sun Oct 21 07:29:04 2012 +0300 @@ -167,6 +167,11 @@ return file->fs->v.write_stream_finish(file, TRUE); } +int fs_write_stream_finish_async(struct fs_file *file) +{ + return file->fs->v.write_stream_finish(file, TRUE); +} + void fs_write_stream_abort(struct fs_file *file, struct ostream **output) { i_assert(*output == file->output); @@ -220,6 +225,11 @@ return src->fs->v.copy(src, dest); } +int fs_copy_finish_async(struct fs_file *dest) +{ + return dest->fs->v.copy(NULL, dest); +} + int fs_rename(struct fs_file *src, struct fs_file *dest) { i_assert(src->fs == dest->fs); diff -r b21fe1a1c7ad -r 260b1ec12f41 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Thu Oct 18 06:58:01 2012 +0300 +++ b/src/lib-fs/fs-api.h Sun Oct 21 07:29:04 2012 +0300 @@ -115,8 +115,9 @@ /* Finish writing via stream. The file will be created/replaced/appended only after this call, same as with fs_write(). Anything written to the stream won't be visible earlier. Returns 1 if ok, 0 if async write isn't finished - yet (retry calling this), -1 if error */ + yet (retry calling fs_write_stream_finish_async()), -1 if error */ int fs_write_stream_finish(struct fs_file *file, struct ostream **output); +int fs_write_stream_finish_async(struct fs_file *file); /* Abort writing via stream. Anything written to the stream is discarded. */ void fs_write_stream_abort(struct fs_file *file, struct ostream **output); @@ -142,6 +143,8 @@ directories are created automatically. Returns 0 if ok, -1 if error occurred. */ int fs_copy(struct fs_file *src, struct fs_file *dest); +/* Try to finish asynchronous fs_copy() */ +int fs_copy_finish_async(struct fs_file *dest); /* Atomically rename a file. Destination parent directories are created automatically. Returns 0 if ok, -1 if error occurred. */ int fs_rename(struct fs_file *src, struct fs_file *dest); From dovecot at dovecot.org Sun Oct 21 08:22:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 21 Oct 2012 08:22:36 +0300 Subject: dovecot-2.2: lib-storage: Fixed deleting mailbox when mail_attri... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a436e4a7503b changeset: 15230:a436e4a7503b user: Timo Sirainen date: Sun Oct 21 08:22:20 2012 +0300 description: lib-storage: Fixed deleting mailbox when mail_attribute_dict wasn't set. diffstat: src/lib-storage/index/index-attribute.c | 16 +++++++++++----- 1 files changed, 11 insertions(+), 5 deletions(-) diffs (40 lines): diff -r 260b1ec12f41 -r a436e4a7503b src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Sun Oct 21 07:29:04 2012 +0300 +++ b/src/lib-storage/index/index-attribute.c Sun Oct 21 08:22:20 2012 +0300 @@ -12,6 +12,7 @@ struct dict_iterate_context *diter; char *prefix; unsigned int prefix_len; + bool dict_disabled; }; static int index_storage_get_dict(struct mailbox *box, struct dict **dict_r, @@ -126,7 +127,10 @@ iter = i_new(struct index_storage_attribute_iter, 1); iter->iter.box = box; - if (index_storage_get_dict(box, &dict, &mailbox_prefix) == 0) { + if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) { + if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTPOSSIBLE) + iter->dict_disabled = TRUE; + } else { iter->prefix = i_strdup(key_get_prefixed(type, mailbox_prefix, prefix)); iter->prefix_len = strlen(iter->prefix); @@ -158,10 +162,12 @@ (struct index_storage_attribute_iter *)_iter; int ret; - ret = iter->diter == NULL ? -1 : - dict_iterate_deinit(&iter->diter); - if (ret < 0) - mail_storage_set_internal_error(_iter->box->storage); + if (iter->diter == NULL) { + ret = iter->dict_disabled ? 0 : -1; + } else { + if ((ret = dict_iterate_deinit(&iter->diter)) < 0) + mail_storage_set_internal_error(_iter->box->storage); + } i_free(iter->prefix); i_free(iter); return ret; From dovecot at dovecot.org Sun Oct 21 10:02:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 21 Oct 2012 10:02:04 +0300 Subject: dovecot-2.2: doveadm: "backup" command is working again. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/454d0563927d changeset: 15231:454d0563927d user: Timo Sirainen date: Sun Oct 21 10:01:54 2012 +0300 description: doveadm: "backup" command is working again. diffstat: src/doveadm/dsync/doveadm-dsync.c | 26 ++- src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 2 + src/doveadm/dsync/dsync-brain-mailbox-tree.c | 11 +- src/doveadm/dsync/dsync-brain-mailbox.c | 47 ++++- src/doveadm/dsync/dsync-brain-mails.c | 42 ++- src/doveadm/dsync/dsync-brain-private.h | 13 + src/doveadm/dsync/dsync-brain.c | 28 ++- src/doveadm/dsync/dsync-brain.h | 4 +- src/doveadm/dsync/dsync-ibc-stream.c | 33 ++- src/doveadm/dsync/dsync-ibc.h | 4 +- src/doveadm/dsync/dsync-mailbox-tree-fill.c | 1 + src/doveadm/dsync/dsync-mailbox-tree-sync.c | 208 +++++++++++++++++---- src/doveadm/dsync/dsync-mailbox-tree.c | 2 + src/doveadm/dsync/dsync-mailbox-tree.h | 16 +- src/doveadm/dsync/test-dsync-mailbox-tree-sync.c | 6 +- 15 files changed, 346 insertions(+), 97 deletions(-) diffs (truncated from 972 to 300 lines): diff -r a436e4a7503b -r 454d0563927d src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 08:22:20 2012 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 10:01:54 2012 +0300 @@ -23,6 +23,8 @@ #include #include +#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:" + struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; enum dsync_brain_sync_type sync_type; @@ -38,7 +40,8 @@ unsigned int lock:1; unsigned int default_replica_location:1; - unsigned int reverse_backup:1; //FIXME + unsigned int backup:1; + unsigned int reverse_backup:1; unsigned int remote:1; }; @@ -315,6 +318,7 @@ struct dsync_ibc *ibc, *ibc2 = NULL; struct dsync_brain *brain; struct mail_namespace *sync_ns = NULL; + enum dsync_brain_flags brain_flags; int ret = 0; user->admin = TRUE; @@ -342,11 +346,16 @@ if (doveadm_debug || doveadm_verbose) { // FIXME } + + brain_flags = DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | + DSYNC_BRAIN_FLAG_SEND_GUID_REQUESTS; + if (ctx->reverse_backup) + brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + else if (ctx->backup) + brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + brain = dsync_brain_master_init(user, ibc, sync_ns, - ctx->sync_type, - DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | - DSYNC_BRAIN_FLAG_SEND_REQUESTS, - ""); + ctx->sync_type, brain_flags, ""); if (!ctx->remote) { if (cmd_dsync_run_local(ctx, user, brain, ibc2) < 0) @@ -469,6 +478,8 @@ ctx->namespace_prefix = optarg; break; case 'R': + if (!ctx->backup) + return FALSE; ctx->reverse_backup = TRUE; break; default: @@ -482,7 +493,7 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "+dEfl:m:n:R"; + ctx->ctx.getopt_args = DSYNC_COMMON_GETOPT_ARGS; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_parse_arg; ctx->ctx.v.preinit = cmd_dsync_preinit; ctx->ctx.v.init = cmd_dsync_init; @@ -498,8 +509,9 @@ struct dsync_cmd_context *ctx; _ctx = cmd_dsync_alloc(); + _ctx->getopt_args = DSYNC_COMMON_GETOPT_ARGS"R"; ctx = (struct dsync_cmd_context *)_ctx; - //FIXME + ctx->backup = TRUE; return _ctx; } diff -r a436e4a7503b -r 454d0563927d src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Sun Oct 21 08:22:20 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Sun Oct 21 10:01:54 2012 +0300 @@ -70,6 +70,8 @@ enum mail_error error; int ret = -1; + i_assert(!brain->backup_send); + switch (change->type) { case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: /* make sure we're deleting the correct mailbox */ diff -r a436e4a7503b -r 454d0563927d src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Sun Oct 21 08:22:20 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Sun Oct 21 10:01:54 2012 +0300 @@ -279,9 +279,18 @@ { struct dsync_mailbox_tree_sync_ctx *ctx; const struct dsync_mailbox_tree_sync_change *change; + enum dsync_mailbox_trees_sync_type sync_type; + + if (brain->backup_send) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL; + else if (brain->backup_recv) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE; + else + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree, - brain->remote_mailbox_tree); + brain->remote_mailbox_tree, + sync_type); while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { if (dsync_brain_mailbox_tree_sync_change(brain, change) < 0) brain->failed = TRUE; diff -r a436e4a7503b -r 454d0563927d src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 08:22:20 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 10:01:54 2012 +0300 @@ -90,11 +90,31 @@ } } +void dsync_brain_sync_init_box_states(struct dsync_brain *brain) +{ + if (brain->backup_send) { + /* we have an exporter, but no importer. */ + brain->box_send_state = DSYNC_BOX_STATE_CHANGES; + brain->box_recv_state = brain->guid_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; + } else if (brain->backup_recv) { + /* we have an importer, but no exporter */ + brain->box_send_state = brain->guid_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_DONE; + brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; + } else { + brain->box_send_state = DSYNC_BOX_STATE_CHANGES; + brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; + } +} + static int dsync_brain_sync_mailbox_init(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_dsync_box, - enum dsync_box_state box_state) + bool wait_for_remote_box) { enum dsync_mailbox_exporter_flags exporter_flags = 0; const struct dsync_mailbox_state *state; @@ -107,8 +127,12 @@ brain->box = box; brain->pre_box_state = brain->state; - brain->box_recv_state = box_state; - brain->box_send_state = box_state; + if (wait_for_remote_box) { + brain->box_send_state = DSYNC_BOX_STATE_MAILBOX; + brain->box_recv_state = DSYNC_BOX_STATE_MAILBOX; + } else { + dsync_brain_sync_init_box_states(brain); + } brain->local_dsync_box = *local_dsync_box; memset(&brain->remote_dsync_box, 0, sizeof(brain->remote_dsync_box)); @@ -143,7 +167,8 @@ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; if (brain->mails_have_guids) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; - brain->box_exporter = + + brain->box_exporter = brain->backup_recv ? NULL : dsync_mailbox_export_init(box, brain->log_scan, last_common_uid, last_common_modseq, exporter_flags); return 0; @@ -181,7 +206,7 @@ if (brain->mails_have_guids) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; - brain->box_importer = + brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->log_scan, last_common_uid, last_common_modseq, remote_dsync_box->uid_next, @@ -290,6 +315,7 @@ dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, struct dsync_mailbox *dsync_box_r) { + enum mailbox_flags flags = 0; struct dsync_mailbox dsync_box; struct mailbox *box; struct dsync_mailbox_node *node; @@ -311,7 +337,11 @@ return -1; } - box = mailbox_alloc(node->ns->list, vname, 0); + if (brain->backup_send) { + /* make sure mailbox isn't modified */ + flags |= MAILBOX_FLAG_READONLY; + } + box = mailbox_alloc(node->ns->list, vname, flags); for (;;) { if ((ret = dsync_box_get(box, &dsync_box)) <= 0) { if (ret < 0) @@ -376,8 +406,7 @@ /* start exporting this mailbox (wait for remote to start importing) */ dsync_ibc_send_mailbox(brain->ibc, &dsync_box); - (void)dsync_brain_sync_mailbox_init(brain, box, &dsync_box, - DSYNC_BOX_STATE_MAILBOX); + (void)dsync_brain_sync_mailbox_init(brain, box, &dsync_box, TRUE); brain->state = DSYNC_STATE_SYNC_MAILS; } @@ -582,7 +611,7 @@ /* start export/import */ if (dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, - DSYNC_BOX_STATE_CHANGES) == 0) + FALSE) == 0) dsync_brain_sync_mailbox_init_remote(brain, dsync_box); brain->state = DSYNC_STATE_SYNC_MAILS; diff -r a436e4a7503b -r 454d0563927d src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Sun Oct 21 08:22:20 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mails.c Sun Oct 21 10:01:54 2012 +0300 @@ -45,10 +45,7 @@ return TRUE; } dsync_brain_sync_mailbox_init_remote(brain, dsync_box); - brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; - brain->box_send_state = DSYNC_BOX_STATE_CHANGES; - - i_assert(brain->box_importer != NULL); + dsync_brain_sync_init_box_states(brain); return TRUE; } @@ -61,8 +58,10 @@ return FALSE; if (ret == DSYNC_IBC_RECV_RET_FINISHED) { dsync_mailbox_import_changes_finish(brain->box_importer); - brain->box_recv_state = brain->guid_requests ? - DSYNC_BOX_STATE_MAIL_REQUESTS : DSYNC_BOX_STATE_MAILS; + if (brain->guid_requests && brain->box_exporter != NULL) + brain->box_recv_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_recv_state = DSYNC_BOX_STATE_MAILS; return TRUE; } dsync_mailbox_import_change(brain->box_importer, change); @@ -78,8 +77,10 @@ return; } dsync_ibc_send_end_of_list(brain->ibc); - brain->box_send_state = brain->guid_requests ? - DSYNC_BOX_STATE_MAIL_REQUESTS : DSYNC_BOX_STATE_MAILS; + if (brain->guid_requests && brain->box_importer != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_send_state = DSYNC_BOX_STATE_MAILS; } static bool dsync_brain_recv_mail_request(struct dsync_brain *brain) @@ -88,11 +89,14 @@ enum dsync_ibc_recv_ret ret; i_assert(brain->guid_requests); + i_assert(brain->box_exporter != NULL); if ((ret = dsync_ibc_recv_mail_request(brain->ibc, &request)) == 0) return FALSE; if (ret == DSYNC_IBC_RECV_RET_FINISHED) { - brain->box_recv_state = DSYNC_BOX_STATE_MAILS; + brain->box_recv_state = brain->box_importer != NULL ? + DSYNC_BOX_STATE_MAILS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; return TRUE; } dsync_mailbox_export_want_mail(brain->box_exporter, request); @@ -111,7 +115,12 @@ } if (brain->box_recv_state > DSYNC_BOX_STATE_CHANGES) { dsync_ibc_send_end_of_list(brain->ibc); - brain->box_send_state = DSYNC_BOX_STATE_MAILS; + if (brain->box_exporter != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAILS; + else { + i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE); + brain->box_send_state = DSYNC_BOX_STATE_DONE; + } } } @@ -126,11 +135,14 @@ return; /* finished with this mailbox */ From dovecot at dovecot.org Sun Oct 21 12:20:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 21 Oct 2012 12:20:47 +0300 Subject: dovecot-2.2: doveadm sync/backup: Added -s parameter to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6c850258002f changeset: 15232:6c850258002f user: Timo Sirainen date: Sun Oct 21 12:20:30 2012 +0300 description: doveadm sync/backup: Added -s parameter to do a fast stateful sync. Initially use an empty string for the input state. The output state is written to stdout. diffstat: src/doveadm/dsync/Makefile.am | 1 - src/doveadm/dsync/doveadm-dsync.c | 19 +++++- src/doveadm/dsync/dsync-brain-mailbox.c | 33 ++------- src/doveadm/dsync/dsync-brain-private.h | 9 +- src/doveadm/dsync/dsync-brain.c | 87 +++++++++++++++++++++---- src/doveadm/dsync/dsync-brain.h | 3 + src/doveadm/dsync/dsync-mailbox-export.c | 20 ++++- src/doveadm/dsync/dsync-mailbox-import.c | 4 +- src/doveadm/dsync/dsync-mailbox-state-export.h | 14 ---- src/doveadm/dsync/dsync-mailbox-state.c | 35 +++++++-- src/doveadm/dsync/dsync-mailbox-state.h | 10 +- src/doveadm/dsync/dsync-mailbox-tree.c | 9 ++ src/doveadm/dsync/dsync-mailbox-tree.h | 5 + 13 files changed, 166 insertions(+), 83 deletions(-) diffs (truncated from 524 to 300 lines): diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/Makefile.am --- a/src/doveadm/dsync/Makefile.am Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/Makefile.am Sun Oct 21 12:20:30 2012 +0300 @@ -41,7 +41,6 @@ dsync-mailbox-import.h \ dsync-mailbox-export.h \ dsync-mailbox-state.h \ - dsync-mailbox-state-export.h \ dsync-mailbox-tree.h \ dsync-mailbox-tree-private.h \ dsync-serializer.h \ diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Sun Oct 21 12:20:30 2012 +0300 @@ -23,12 +23,13 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:" +#define DSYNC_COMMON_GETOPT_ARGS "+dEfl:m:n:s:" struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; enum dsync_brain_sync_type sync_type; const char *mailbox, *namespace_prefix; + const char *state_input; const char *remote_name; const char *local_location; @@ -355,7 +356,9 @@ brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; brain = dsync_brain_master_init(user, ibc, sync_ns, - ctx->sync_type, brain_flags, ""); + ctx->sync_type, brain_flags, + ctx->state_input == NULL ? "" : + ctx->state_input); if (!ctx->remote) { if (cmd_dsync_run_local(ctx, user, brain, ibc2) < 0) @@ -364,6 +367,12 @@ cmd_dsync_run_remote(user); } + if (ctx->state_input != NULL) { + string_t *str = t_str_new(128); + dsync_brain_get_state(brain, str); + printf("%s\n", str_c(str)); + } + if (dsync_brain_deinit(&brain) < 0) _ctx->exit_code = EX_TEMPFAIL; dsync_ibc_deinit(&ibc); @@ -482,6 +491,12 @@ return FALSE; ctx->reverse_backup = TRUE; break; + case 's': + if (ctx->sync_type != DSYNC_BRAIN_SYNC_TYPE_FULL && + *optarg != '\0') + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE; + ctx->state_input = optarg; + break; default: return FALSE; } diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Sun Oct 21 12:20:30 2012 +0300 @@ -63,31 +63,21 @@ dsync_mailbox_state_find(struct dsync_brain *brain, const guid_128_t mailbox_guid) { - const struct dsync_mailbox_state *state; + const uint8_t *guid_p; - array_foreach(&brain->mailbox_states, state) { - if (memcmp(state->mailbox_guid, mailbox_guid, - sizeof(state->mailbox_guid)) == 0) - return state; - } - return NULL; + guid_p = mailbox_guid; + return hash_table_lookup(brain->mailbox_states, guid_p); } static void dsync_mailbox_state_remove(struct dsync_brain *brain, const guid_128_t mailbox_guid) { - const struct dsync_mailbox_state *states; - unsigned int i, count; + const uint8_t *guid_p; - states = array_get(&brain->mailbox_states, &count); - for (i = 0; i < count; i++) { - if (memcmp(states[i].mailbox_guid, mailbox_guid, - sizeof(states[i].mailbox_guid)) == 0) { - array_delete(&brain->mailbox_states, i, 1); - break; - } - } + guid_p = mailbox_guid; + if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL) + hash_table_remove(brain->mailbox_states, guid_p); } void dsync_brain_sync_init_box_states(struct dsync_brain *brain) @@ -217,16 +207,9 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) { - struct dsync_mailbox_state *state; - uint8_t *guid_p; - i_assert(brain->box != NULL); - state = p_new(brain->pool, struct dsync_mailbox_state, 1); - *state = brain->mailbox_state; - guid_p = state->mailbox_guid; - hash_table_insert(brain->remote_mailbox_states, guid_p, state); - + array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1); if (brain->box_exporter != NULL) { const char *error; diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Sun Oct 21 12:20:30 2012 +0300 @@ -66,15 +66,14 @@ /* list of mailbox states for master brain: given to brain at init and for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */ - ARRAY_TYPE(dsync_mailbox_state) mailbox_states; + HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states; /* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */ - unsigned int mailbox_state_idx; + struct hash_iterate_context *mailbox_states_iter; /* state of the mailbox we're currently syncing, changed at init and deinit */ struct dsync_mailbox_state mailbox_state; - /* GUID -> dsync_mailbox_state for mailboxes that have already - been synced */ - HASH_TABLE(uint8_t *, struct dsync_mailbox_state *) remote_mailbox_states; + /* new states for synced mailboxes */ + ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states; unsigned int master_brain:1; unsigned int guid_requests:1; diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Sun Oct 21 12:20:30 2012 +0300 @@ -47,9 +47,9 @@ brain->user = user; brain->ibc = ibc; brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN; - hash_table_create(&brain->remote_mailbox_states, - brain->pool, 0, guid_128_hash, guid_128_cmp); - p_array_init(&brain->mailbox_states, pool, 64); + hash_table_create(&brain->mailbox_states, pool, 0, + guid_128_hash, guid_128_cmp); + p_array_init(&brain->remote_mailbox_states, pool, 64); return brain; } @@ -87,12 +87,14 @@ brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; if (sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) { - if (dsync_mailbox_states_import(&brain->mailbox_states, state, + if (dsync_mailbox_states_import(brain->mailbox_states, + brain->pool, state, &error) < 0) { - array_clear(&brain->mailbox_states); + hash_table_clear(brain->mailbox_states, FALSE); i_error("Saved sync state is invalid, " "falling back to full sync: %s", error); - brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + brain->sync_type = sync_type = + DSYNC_BRAIN_SYNC_TYPE_FULL; } else { brain->state = DSYNC_STATE_MASTER_SEND_LAST_COMMON; } @@ -142,8 +144,9 @@ dsync_brain_sync_mailbox_deinit(brain); if (brain->local_tree_iter != NULL) dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); - - hash_table_destroy(&brain->remote_mailbox_states); + if (brain->mailbox_states_iter != NULL) + hash_table_iterate_deinit(&brain->mailbox_states_iter); + hash_table_destroy(&brain->mailbox_states); ret = brain->failed ? -1 : 0; pool_unref(&brain->pool); @@ -184,22 +187,41 @@ static void dsync_brain_master_send_last_common(struct dsync_brain *brain) { - const struct dsync_mailbox_state *states; - unsigned int count; + struct dsync_mailbox_state *state; + uint8_t *guid; enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK; i_assert(brain->master_brain); - states = array_get(&brain->mailbox_states, &count); - while (brain->mailbox_state_idx < count) { + if (brain->mailbox_states_iter == NULL) { + brain->mailbox_states_iter = + hash_table_iterate_init(brain->mailbox_states); + } + + for (;;) { if (ret == DSYNC_IBC_SEND_RET_FULL) return; - ret = dsync_ibc_send_mailbox_state(brain->ibc, - &states[brain->mailbox_state_idx++]); + if (!hash_table_iterate(brain->mailbox_states_iter, + brain->mailbox_states, &guid, &state)) + break; + ret = dsync_ibc_send_mailbox_state(brain->ibc, state); } + hash_table_iterate_deinit(&brain->mailbox_states_iter); + dsync_ibc_send_end_of_list(brain->ibc); brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; - brain->mailbox_state_idx = 0; +} + +static void dsync_mailbox_state_add(struct dsync_brain *brain, + const struct dsync_mailbox_state *state) +{ + struct dsync_mailbox_state *dupstate; + uint8_t *guid_p; + + dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1); + *dupstate = *state; + guid_p = dupstate->mailbox_guid; + hash_table_insert(brain->mailbox_states, guid_p, dupstate); } static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain) @@ -211,7 +233,7 @@ i_assert(!brain->master_brain); while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) { - array_append(&brain->mailbox_states, &state, 1); + dsync_mailbox_state_add(brain, &state); changed = TRUE; } if (ret == DSYNC_IBC_RECV_RET_FINISHED) { @@ -292,6 +314,39 @@ return ret; } +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_node *node; + const struct dsync_mailbox_state *new_state; + struct dsync_mailbox_state *state; + const uint8_t *guid_p; + uint8_t *guid; + + /* update mailbox states */ + array_foreach(&brain->remote_mailbox_states, new_state) { + guid_p = new_state->mailbox_guid; + state = hash_table_lookup(brain->mailbox_states, guid_p); + if (state != NULL) + *state = *new_state; + else + dsync_mailbox_state_add(brain, new_state); + } + + /* remove nonexistent mailboxes */ + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree, + guid); + if (node == NULL || + node->existence != DSYNC_MAILBOX_NODE_EXISTS) + hash_table_remove(brain->mailbox_states, guid); + } + hash_table_iterate_deinit(&iter); + + dsync_mailbox_states_export(brain->mailbox_states, output); +} + bool dsync_brain_has_failed(struct dsync_brain *brain) { return brain->failed; diff -r 454d0563927d -r 6c850258002f src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Sun Oct 21 10:01:54 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Sun Oct 21 12:20:30 2012 +0300 @@ -40,5 +40,8 @@ bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r); /* Returns TRUE if brain has failed, and there's no point in continuing. */ bool dsync_brain_has_failed(struct dsync_brain *brain); From dovecot at dovecot.org Mon Oct 22 15:17:50 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:17:50 +0300 Subject: dovecot-2.1: director: Fixed previous broken change for handling... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/e4c337f38ed6 changeset: 14774:e4c337f38ed6 user: Timo Sirainen date: Mon Oct 22 15:17:39 2012 +0300 description: director: Fixed previous broken change for handling USER-WEAK commands. diffstat: src/director/director-connection.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r a97c62fdd4ad -r e4c337f38ed6 src/director/director-connection.c --- a/src/director/director-connection.c Sun Oct 21 07:13:44 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:17:39 2012 +0300 @@ -725,7 +725,9 @@ bool weak = TRUE; int ret; - if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) != 0) + /* note that unlike other commands we don't want to just ignore + duplicate commands */ + if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) return ret > 0; if (str_array_length(args) != 2 || From dovecot at dovecot.org Mon Oct 22 15:21:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:21:04 +0300 Subject: dovecot-2.1: director: Don't handle pending requests from all ar... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/cd646623a1a8 changeset: 14775:cd646623a1a8 user: Timo Sirainen date: Mon Oct 22 15:20:57 2012 +0300 description: director: Don't handle pending requests from all around the code. I'm not sure if this actually fixes any bugs, but it definitely makes the state cleaner. diffstat: src/director/director.c | 15 ++++++++++++++- src/director/director.h | 1 + 2 files changed, 15 insertions(+), 1 deletions(-) diffs (45 lines): diff -r e4c337f38ed6 -r cd646623a1a8 src/director/director.c --- a/src/director/director.c Mon Oct 22 15:17:39 2012 +0300 +++ b/src/director/director.c Mon Oct 22 15:20:57 2012 +0300 @@ -787,9 +787,20 @@ user->username_hash)); } +static void director_state_callback_timeout(struct director *dir) +{ + timeout_remove(&dir->to_callback); + dir->state_change_callback(dir); +} + void director_set_state_changed(struct director *dir) { - dir->state_change_callback(dir); + /* we may get called to here from various places. use a timeout to + make sure the state callback is called with a clean state. */ + if (dir->to_callback == NULL) { + dir->to_callback = + timeout_add(0, director_state_callback_timeout, dir); + } } void director_update_send(struct director *dir, struct director_host *src, @@ -866,6 +877,8 @@ timeout_remove(&dir->to_sync); if (dir->to_remove_dirs != NULL) timeout_remove(&dir->to_remove_dirs); + if (dir->to_callback != NULL) + timeout_remove(&dir->to_callback); while (array_count(&dir->dir_hosts) > 0) { hostp = array_idx(&dir->dir_hosts, 0); host = *hostp; diff -r e4c337f38ed6 -r cd646623a1a8 src/director/director.h --- a/src/director/director.h Mon Oct 22 15:17:39 2012 +0300 +++ b/src/director/director.h Mon Oct 22 15:20:57 2012 +0300 @@ -44,6 +44,7 @@ ARRAY_DEFINE(connections, struct director_connection *); struct timeout *to_reconnect; struct timeout *to_sync; + struct timeout *to_callback; /* current mail hosts */ struct mail_host_list *mail_hosts; From dovecot at dovecot.org Mon Oct 22 15:24:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:24:01 +0300 Subject: dovecot-2.1: director: Log more clearly why a request timeouts. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/10fae591707c changeset: 14776:10fae591707c user: Timo Sirainen date: Mon Oct 22 15:23:25 2012 +0300 description: director: Log more clearly why a request timeouts. diffstat: src/director/director-request.c | 51 +++++++++++++++++++++++++++++++++------- 1 files changed, 42 insertions(+), 9 deletions(-) diffs (156 lines): diff -r cd646623a1a8 -r 10fae591707c src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:20:57 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:23:25 2012 +0300 @@ -12,24 +12,44 @@ #define DIRECTOR_REQUEST_TIMEOUT_SECS 30 #define RING_NOCONN_WARNING_DELAY_MSECS (2*1000) +enum director_request_delay_reason { + REQUEST_DELAY_NONE = 0, + REQUEST_DELAY_RINGNOTHANDSHAKED, + REQUEST_DELAY_RINGNOTSYNCED, + REQUEST_DELAY_NOHOSTS, + REQUEST_DELAY_WEAK, + REQUEST_DELAY_KILL +}; + +static const char *delay_reason_strings[] = { + "unknown", + "ring not handshaked", + "ring not synced", + "no hosts", + "weak user", + "kill waiting" +}; + struct director_request { struct director *dir; time_t create_time; unsigned int username_hash; + enum director_request_delay_reason delay_reason; director_request_callback *callback; void *context; }; static const char * -director_request_get_timeout_error(struct director_request *request) +director_request_get_timeout_error(struct director_request *request, + struct user *user, string_t *str) { - string_t *str = t_str_new(128); - struct user *user; unsigned int secs; - str_printfa(str, "Timeout - queued for %u secs (", + str_truncate(str, 0); + str_printfa(str, "Timeout because %s - queued for %u secs (", + delay_reason_strings[request->delay_reason], (unsigned int)(ioloop_time - request->create_time)); if (request->dir->ring_last_sync_time == 0) @@ -42,8 +62,6 @@ str_printfa(str, "Ring not synced for %u secs", secs); } - user = user_directory_lookup(request->dir->users, - request->username_hash); if (user != NULL) { if (user->weak) str_append(str, ", weak user"); @@ -57,7 +75,9 @@ static void director_request_timeout(struct director *dir) { struct director_request **requestp, *request; + struct user *user; const char *errormsg; + string_t *str = t_str_new(128); while (array_count(&dir->pending_requests) > 0) { requestp = array_idx_modifiable(&dir->pending_requests, 0); @@ -67,8 +87,12 @@ DIRECTOR_REQUEST_TIMEOUT_SECS > ioloop_time) break; + user = user_directory_lookup(request->dir->users, + request->username_hash); + errormsg = director_request_get_timeout_error(request, + user, str); + array_delete(&dir->pending_requests, 0, 1); - errormsg = director_request_get_timeout_error(request); T_BEGIN { request->callback(NULL, errormsg, request->context); } T_END; @@ -127,13 +151,16 @@ ring_noconn_warning, dir); } -static bool director_request_existing(struct director *dir, struct user *user) +static bool +director_request_existing(struct director_request *request, struct user *user) { + struct director *dir = request->dir; struct mail_host *host; if (user->kill_state != USER_KILL_STATE_NONE) { /* delay processing this user's connections until its existing connections have been killed */ + request->delay_reason = REQUEST_DELAY_KILL; return FALSE; } if (dir->right == NULL && dir->ring_synced) { @@ -147,6 +174,7 @@ if (user->weak) { /* wait for user to become non-weak */ + request->delay_reason = REQUEST_DELAY_WEAK; return FALSE; } if (!user_directory_user_is_near_expiring(dir->users, user)) @@ -157,6 +185,7 @@ host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash); if (!dir->ring_synced) { /* try again later once ring is synced */ + request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; return FALSE; } if (user->host == host) { @@ -200,6 +229,7 @@ } else { user->weak = TRUE; director_update_user_weak(dir, dir->self_host, NULL, user); + request->delay_reason = REQUEST_DELAY_WEAK; return FALSE; } } @@ -213,24 +243,27 @@ if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ ring_log_delayed_warning(dir); + request->delay_reason = REQUEST_DELAY_RINGNOTHANDSHAKED; return FALSE; } user = user_directory_lookup(dir->users, request->username_hash); if (user != NULL) { - if (!director_request_existing(dir, user)) + if (!director_request_existing(request, user)) return FALSE; user_directory_refresh(dir->users, user); } else { if (!dir->ring_synced) { /* delay adding new users until ring is again synced */ ring_log_delayed_warning(dir); + request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; return FALSE; } host = mail_host_get_by_hash(dir->mail_hosts, request->username_hash); if (host == NULL) { /* all hosts have been removed */ + request->delay_reason = REQUEST_DELAY_NOHOSTS; return FALSE; } user = user_directory_add(dir->users, request->username_hash, From dovecot at dovecot.org Mon Oct 22 15:29:38 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:29:38 +0300 Subject: dovecot-2.1: director: If user's weak-flag appears to have gotte... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7352d48b4071 changeset: 14777:7352d48b4071 user: Timo Sirainen date: Mon Oct 22 15:29:27 2012 +0300 description: director: If user's weak-flag appears to have gotten stuck, unset it. diffstat: src/director/director-request.c | 7 +++++++ src/director/user-directory.c | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletions(-) diffs (44 lines): diff -r 10fae591707c -r 7352d48b4071 src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:23:25 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:29:27 2012 +0300 @@ -91,6 +91,13 @@ request->username_hash); errormsg = director_request_get_timeout_error(request, user, str); + if (user != NULL && + request->delay_reason == REQUEST_DELAY_WEAK) { + /* weakness appears to have gotten stuck. this is a + bug, but try to fix it for future requests by + removing the weakness. */ + user->weak = FALSE; + } array_delete(&dir->pending_requests, 0, 1); T_BEGIN { diff -r 10fae591707c -r 7352d48b4071 src/director/user-directory.c --- a/src/director/user-directory.c Mon Oct 22 15:23:25 2012 +0300 +++ b/src/director/user-directory.c Mon Oct 22 15:29:27 2012 +0300 @@ -65,7 +65,22 @@ { time_t expire_timestamp = user->timestamp + dir->timeout_secs; - return expire_timestamp >= ioloop_time; + if (expire_timestamp >= ioloop_time) + return TRUE; + + if (user->kill_state != USER_KILL_STATE_NONE) { + /* don't free this user until the kill is finished */ + return TRUE; + } + + if (user->weak) { + if (expire_timestamp + USER_NEAR_EXPIRING_MAX >= ioloop_time) + return TRUE; + + i_warning("User %u weakness appears to be stuck, removing it", + user->username_hash); + } + return FALSE; } static void user_directory_drop_expired(struct user_directory *dir) From dovecot at dovecot.org Mon Oct 22 15:30:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:30:06 +0300 Subject: dovecot-2.1: director: Minor code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/1dcf0090648e changeset: 14778:1dcf0090648e user: Timo Sirainen date: Mon Oct 22 15:30:01 2012 +0300 description: director: Minor code cleanup diffstat: src/director/director-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7352d48b4071 -r 1dcf0090648e src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 22 15:29:27 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:30:01 2012 +0300 @@ -728,7 +728,7 @@ /* note that unlike other commands we don't want to just ignore duplicate commands */ if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) - return ret > 0; + return FALSE; if (str_array_length(args) != 2 || str_to_uint(args[0], &username_hash) < 0 || From dovecot at dovecot.org Mon Oct 22 15:32:12 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:32:12 +0300 Subject: dovecot-2.1: director: Don't remove user's weak flag from notify... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/c16a0182533f changeset: 14779:c16a0182533f user: Timo Sirainen date: Mon Oct 22 15:32:04 2012 +0300 description: director: Don't remove user's weak flag from notify connection. If notify connection worked properly, the weak flag should never have been set in the first place. And if it's just suddenly removed, it won't finish the pending requests properly. diffstat: src/director/notify-connection.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 1dcf0090648e -r c16a0182533f src/director/notify-connection.c --- a/src/director/notify-connection.c Mon Oct 22 15:30:01 2012 +0300 +++ b/src/director/notify-connection.c Mon Oct 22 15:32:04 2012 +0300 @@ -33,7 +33,6 @@ i_warning("notify: User %s refreshed too late " "(%d secs)", line, diff); } - user->weak = FALSE; user_directory_refresh(conn->dir->users, user); director_update_user(conn->dir, conn->dir->self_host, user); From dovecot at dovecot.org Mon Oct 22 15:36:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 15:36:06 +0300 Subject: dovecot-2.1: director: -D parameter now enables extensive debug ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/48af47f2eb9c changeset: 14780:48af47f2eb9c user: Timo Sirainen date: Mon Oct 22 15:35:59 2012 +0300 description: director: -D parameter now enables extensive debug logging. diffstat: src/director/director-connection.c | 50 +++++++++++++++++++++---------------- src/director/director-request.c | 17 ++++++++++++ src/director/director.c | 33 ++++++++++++++++-------- src/director/director.h | 5 +++- src/director/main.c | 2 +- 5 files changed, 72 insertions(+), 35 deletions(-) diffs (truncated from 305 to 300 lines): diff -r c16a0182533f -r 48af47f2eb9c src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:35:59 2012 +0300 @@ -454,12 +454,15 @@ *user_r = user_directory_add(dir->users, username_hash, host, timestamp); (*user_r)->weak = weak; + dir_debug("user refresh: %u added", username_hash); return TRUE; } if (user->weak) { if (!weak) { /* removing user's weakness */ + dir_debug("user refresh: %u weakness removed", + username_hash); unset_weak_user = TRUE; user->weak = FALSE; ret = TRUE; @@ -469,6 +472,7 @@ } else if (weak && !user_directory_user_is_recently_updated(dir->users, user)) { /* mark the user as weak */ + dir_debug("user refresh: %u set weak", username_hash); user->weak = TRUE; ret = TRUE; } else if (user->host != host) { @@ -516,6 +520,8 @@ user_directory_refresh(dir->users, user); ret = TRUE; } + dir_debug("user refresh: %u refreshed timeout to %ld", + username_hash, (long)user->timestamp); if (unset_weak_user) { /* user is no longer weak. handle pending requests for @@ -938,7 +944,7 @@ unsigned int handshake_secs = time(NULL) - conn->created; string_t *str; - if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || dir->debug) { + if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || director_debug) { str = t_str_new(128); str_printfa(str, "director(%s): Handshake took %u secs, " "bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T, @@ -1065,10 +1071,8 @@ /* duplicate SYNC (which was sent just in case the previous one got lost) */ } else { - if (dir->debug) { - i_debug("Ring is synced (%s sent seq=%u)", - conn->name, seq); - } + dir_debug("Ring is synced (%s sent seq=%u)", + conn->name, seq); director_set_ring_synced(dir); } } else if (dir->right != NULL) { @@ -1130,23 +1134,18 @@ director_host_cmp_to_self(host, dir->right->host, dir->self_host) <= 0) { /* the old connection is the correct one */ - if (dir->debug) { - i_debug("Ignoring CONNECT request to %s " - "(current right is %s)", - host->name, dir->right->name); - } + dir_debug("Ignoring CONNECT request to %s (current right is %s)", + host->name, dir->right->name); return TRUE; } - if (dir->debug) { - if (dir->right == NULL) { - i_debug("Received CONNECT request to %s, " - "initializing right", host->name); - } else { - i_debug("Received CONNECT request to %s, " - "replacing current right %s", - host->name, dir->right->name); - } + if (dir->right == NULL) { + dir_debug("Received CONNECT request to %s, " + "initializing right", host->name); + } else { + dir_debug("Received CONNECT request to %s, " + "replacing current right %s", + host->name, dir->right->name); } /* connect here */ @@ -1254,6 +1253,8 @@ const char *cmd, *const *args; bool ret; + dir_debug("input: %s: %s", conn->name, line); + args = t_strsplit_tab(line); cmd = args[0]; args++; if (cmd == NULL) { @@ -1535,9 +1536,9 @@ *_conn = NULL; - if (dir->debug && conn->host != NULL) { - i_debug("Disconnecting from %s: %s", - conn->host->name, remote_reason); + if (conn->host != NULL) { + dir_debug("Disconnecting from %s: %s", + conn->host->name, remote_reason); } if (*remote_reason != '\0' && conn->minor_version >= DIRECTOR_VERSION_QUIT) { @@ -1629,6 +1630,11 @@ if (conn->output->closed || !conn->connected) return; + if (director_debug) T_BEGIN { + const char *const *lines = t_strsplit(data, "\n"); + for (; lines[1] != NULL; lines++) + dir_debug("output: %s: %s", conn->name, *lines); + } T_END; ret = o_stream_send(conn->output, data, len); if (ret != (off_t)len) { if (ret < 0) diff -r c16a0182533f -r 48af47f2eb9c src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:35:59 2012 +0300 @@ -168,6 +168,8 @@ /* delay processing this user's connections until its existing connections have been killed */ request->delay_reason = REQUEST_DELAY_KILL; + dir_debug("request: %u waiting for kill to finish", + user->username_hash); return FALSE; } if (dir->right == NULL && dir->ring_synced) { @@ -182,6 +184,8 @@ if (user->weak) { /* wait for user to become non-weak */ request->delay_reason = REQUEST_DELAY_WEAK; + dir_debug("request: %u waiting for weakness", + request->username_hash); return FALSE; } if (!user_directory_user_is_near_expiring(dir->users, user)) @@ -193,6 +197,8 @@ if (!dir->ring_synced) { /* try again later once ring is synced */ request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; + dir_debug("request: %u waiting for sync for making weak", + request->username_hash); return FALSE; } if (user->host == host) { @@ -237,6 +243,7 @@ user->weak = TRUE; director_update_user_weak(dir, dir->self_host, NULL, user); request->delay_reason = REQUEST_DELAY_WEAK; + dir_debug("request: %u set to weak", request->username_hash); return FALSE; } } @@ -249,6 +256,8 @@ if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ + dir_debug("request: %u waiting for handshake", + request->username_hash); ring_log_delayed_warning(dir); request->delay_reason = REQUEST_DELAY_RINGNOTHANDSHAKED; return FALSE; @@ -259,11 +268,15 @@ if (!director_request_existing(request, user)) return FALSE; user_directory_refresh(dir->users, user); + dir_debug("request: %u refreshed timeout to %u", + request->username_hash, user->timestamp); } else { if (!dir->ring_synced) { /* delay adding new users until ring is again synced */ ring_log_delayed_warning(dir); request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; + dir_debug("request: %u waiting for sync for adding", + request->username_hash); return FALSE; } host = mail_host_get_by_hash(dir->mail_hosts, @@ -271,10 +284,14 @@ if (host == NULL) { /* all hosts have been removed */ request->delay_reason = REQUEST_DELAY_NOHOSTS; + dir_debug("request: %u waiting for hosts", + request->username_hash); return FALSE; } user = user_directory_add(dir->users, request->username_hash, host, ioloop_time); + dir_debug("request: %u added timeout to %u", + request->username_hash, user->timestamp); } i_assert(!user->weak); diff -r c16a0182533f -r 48af47f2eb9c src/director/director.c --- a/src/director/director.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director.c Mon Oct 22 15:35:59 2012 +0300 @@ -22,6 +22,8 @@ #define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000 #define DIRECTOR_DELAYED_DIR_REMOVE_MSECS (1000*30) +bool director_debug; + static bool director_is_self_ip_set(struct director *dir) { struct ip_addr ip; @@ -106,10 +108,8 @@ if (director_has_outgoing_connection(dir, host)) return 0; - if (dir->debug) { - i_debug("Connecting to %s:%u", - net_ip2addr(&host->ip), host->port); - } + dir_debug("Connecting to %s:%u", + net_ip2addr(&host->ip), host->port); port = dir->test_port != 0 ? dir->test_port : host->port; fd = net_connect_ip(&host->ip, port, &dir->self_ip); if (fd == -1) { @@ -239,8 +239,7 @@ "continuing delayed requests"); dir->ring_handshake_warning_sent = FALSE; } - if (dir->debug) - i_debug("Director ring handshaked"); + dir_debug("Director ring handshaked"); dir->ring_handshaked = TRUE; director_set_ring_synced(dir); @@ -383,11 +382,9 @@ return; } - if (dir->debug) { - i_debug("Ring is desynced (seq=%u, sending SYNC to %s)", - dir->sync_seq, dir->right == NULL ? "(nowhere)" : - director_connection_get_name(dir->right)); - } + dir_debug("Ring is desynced (seq=%u, sending SYNC to %s)", + dir->sync_seq, dir->right == NULL ? "(nowhere)" : + director_connection_get_name(dir->right)); /* send PINGs to our connections more rapidly until we've synced again. if the connection has actually died, we don't need to wait (and @@ -889,3 +886,17 @@ array_free(&dir->connections); i_free(dir); } + +void dir_debug(const char *fmt, ...) +{ + va_list args; + + if (!director_debug) + return; + + va_start(args, fmt); + T_BEGIN { + i_debug("%s", t_strdup_vprintf(fmt, args)); + } T_END; + va_end(args); +} diff -r c16a0182533f -r 48af47f2eb9c src/director/director.h --- a/src/director/director.h Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director.h Mon Oct 22 15:35:59 2012 +0300 @@ -80,9 +80,10 @@ unsigned int ring_synced:1; unsigned int sync_frozen:1; unsigned int sync_pending:1; - unsigned int debug:1; }; +extern bool director_debug; + /* Create a new director. If listen_ip specifies an actual IP, it's used with listen_port for finding ourself from the director_servers setting. listen_port is used regardless by director_host_add_from_string() for hosts @@ -147,4 +148,6 @@ int director_connect_host(struct director *dir, struct director_host *host); +void dir_debug(const char *fmt, ...) ATTR_FORMAT(1, 2); + #endif diff -r c16a0182533f -r 48af47f2eb9c src/director/main.c --- a/src/director/main.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/main.c Mon Oct 22 15:35:59 2012 +0300 @@ -215,7 +215,7 @@ main_preinit(); master_service_init_finish(master_service); director->test_port = test_port; From dovecot at dovecot.org Mon Oct 22 18:36:07 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 18:36:07 +0300 Subject: dovecot-2.1: decode2text.sh: Assume xmlunzip exists in the same ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/09ed39c15584 changeset: 14781:09ed39c15584 user: Timo Sirainen date: Mon Oct 22 18:35:56 2012 +0300 description: decode2text.sh: Assume xmlunzip exists in the same directory as this script. This avoids hardcoding /usr/local/libexec/dovecot/ path in it. diffstat: src/plugins/fts/decode2text.sh | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 48af47f2eb9c -r 09ed39c15584 src/plugins/fts/decode2text.sh --- a/src/plugins/fts/decode2text.sh Mon Oct 22 15:35:59 2012 +0300 +++ b/src/plugins/fts/decode2text.sh Mon Oct 22 18:35:56 2012 +0300 @@ -17,6 +17,7 @@ # } # } +libexec_dir=`dirname $0` content_type=$1 # The second parameter is the format's filename extension, which is used when @@ -66,7 +67,7 @@ cd $tempdir || exit 1 unzip -q "$path" 2>/dev/null || exit 0 find . -name "$name" -print0 | xargs -0 cat | - /usr/local/libexec/dovecot/xml2text + $libexec_dir/xml2text } wait_timeout() { From dovecot at dovecot.org Mon Oct 22 18:59:30 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Oct 2012 18:59:30 +0300 Subject: dovecot-2.1: auth: Log a nicer message if client timeouts authen... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/49bb6cc43d03 changeset: 14782:49bb6cc43d03 user: Timo Sirainen date: Mon Oct 22 18:59:20 2012 +0300 description: auth: Log a nicer message if client timeouts authentication in the middle. diffstat: src/auth/auth-request-handler.c | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-) diffs (29 lines): diff -r 09ed39c15584 -r 49bb6cc43d03 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Mon Oct 22 18:35:56 2012 +0300 +++ b/src/auth/auth-request-handler.c Mon Oct 22 18:59:20 2012 +0300 @@ -401,19 +401,18 @@ static void auth_request_timeout(struct auth_request *request) { - const char *str; + unsigned int secs = (unsigned int)(time(NULL) - request->last_access); - str = t_strdup_printf("Request %u.%u timeouted after %u secs, state=%d", - request->handler->client_pid, request->id, - (unsigned int)(time(NULL) - request->last_access), - request->state); if (request->state != AUTH_REQUEST_STATE_MECH_CONTINUE) { /* client's fault */ auth_request_log_error(request, request->mech->mech_name, - "%s", str); + "Request %u.%u timed out after %u secs, state=%d", + request->handler->client_pid, request->id, + secs, request->state); } else if (request->set->verbose) { auth_request_log_info(request, request->mech->mech_name, - "%s", str); + "Request timed out waiting for client to continue authentication " + "(%u secs)", secs); } auth_request_handler_remove(request->handler, request); } From dovecot at dovecot.org Tue Oct 23 12:34:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 12:34:53 +0300 Subject: dovecot-2.2: dsync: Use mailbox_move() to reassign UIDs instead ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f417cab3a21f changeset: 15233:f417cab3a21f user: Timo Sirainen date: Tue Oct 23 12:34:42 2012 +0300 description: dsync: Use mailbox_move() to reassign UIDs instead of copy+expunge. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diffs (31 lines): diff -r 6c850258002f -r f417cab3a21f src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Sun Oct 21 12:20:30 2012 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue Oct 23 12:34:42 2012 +0300 @@ -966,10 +966,8 @@ save_ctx = mailbox_save_alloc(importer->ext_trans); mailbox_save_copy_flags(save_ctx, importer->mail); mailbox_save_set_uid(save_ctx, new_uid); - if (mailbox_copy(&save_ctx, importer->mail) == 0) { + if (mailbox_move(&save_ctx, importer->mail) == 0) array_append(&importer->wanted_uids, &new_uid, 1); - mail_expunge(importer->mail); - } } static void @@ -1266,10 +1264,12 @@ save_ctx = mailbox_save_alloc(trans); mailbox_save_copy_flags(save_ctx, mail); - if (mailbox_copy(&save_ctx, mail) < 0) + if (mailbox_move(&save_ctx, mail) < 0) { + i_error("Couldn't move mail within mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); ret = -1; - else - mail_expunge(mail); + } } mail_free(&mail); From dovecot at dovecot.org Tue Oct 23 12:59:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 12:59:46 +0300 Subject: dovecot-2.2: connection API: Track the number of connections. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/acd76b5272e9 changeset: 15234:acd76b5272e9 user: Timo Sirainen date: Tue Oct 23 12:59:41 2012 +0300 description: connection API: Track the number of connections. diffstat: src/lib/connection.c | 6 ++++++ src/lib/connection.h | 1 + 2 files changed, 7 insertions(+), 0 deletions(-) diffs (48 lines): diff -r f417cab3a21f -r acd76b5272e9 src/lib/connection.c --- a/src/lib/connection.c Tue Oct 23 12:34:42 2012 +0300 +++ b/src/lib/connection.c Tue Oct 23 12:59:41 2012 +0300 @@ -163,6 +163,7 @@ connection_init_streams(conn); DLLIST_PREPEND(&list->connections, conn); + list->connections_count++; } void connection_init_client_ip(struct connection_list *list, @@ -179,6 +180,7 @@ conn->port = port; DLLIST_PREPEND(&list->connections, conn); + list->connections_count++; } void connection_init_client_unix(struct connection_list *list, @@ -191,6 +193,7 @@ conn->name = i_strdup(path); DLLIST_PREPEND(&list->connections, conn); + list->connections_count++; } static void connection_ip_connected(struct connection *conn) @@ -253,6 +256,9 @@ void connection_deinit(struct connection *conn) { + i_assert(conn->list->connections_count > 0); + + conn->list->connections_count--; DLLIST_REMOVE(&conn->list->connections, conn); connection_disconnect(conn); diff -r f417cab3a21f -r acd76b5272e9 src/lib/connection.h --- a/src/lib/connection.h Tue Oct 23 12:34:42 2012 +0300 +++ b/src/lib/connection.h Tue Oct 23 12:59:41 2012 +0300 @@ -85,6 +85,7 @@ struct connection_list { struct connection *connections; + unsigned int connections_count; struct connection_settings set; struct connection_vfuncs v; From dovecot at dovecot.org Tue Oct 23 20:16:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:16:21 +0300 Subject: dovecot-2.2: imap: URLFETCH error handling fixes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3e70eacf67a4 changeset: 15235:3e70eacf67a4 user: Timo Sirainen date: Tue Oct 23 20:07:06 2012 +0300 description: imap: URLFETCH error handling fixes. diffstat: src/imap/cmd-urlfetch.c | 60 ++++++++++++++++++------------ src/lib-imap-urlauth/imap-urlauth-fetch.c | 17 ++++---- src/lib-imap-urlauth/imap-urlauth-fetch.h | 3 +- 3 files changed, 46 insertions(+), 34 deletions(-) diffs (191 lines): diff -r acd76b5272e9 -r 3e70eacf67a4 src/imap/cmd-urlfetch.c --- a/src/imap/cmd-urlfetch.c Tue Oct 23 12:59:41 2012 +0300 +++ b/src/imap/cmd-urlfetch.c Tue Oct 23 20:07:06 2012 +0300 @@ -16,6 +16,7 @@ struct cmd_urlfetch_context { struct imap_urlauth_fetch *ufetch; struct istream *input; + uoff_t size; unsigned int failed:1; unsigned int finished:1; @@ -28,8 +29,7 @@ enum imap_urlauth_fetch_flags flags; }; -static void cmd_urlfetch_finish -(struct client_command_context *cmd) +static void cmd_urlfetch_finish(struct client_command_context *cmd) { struct cmd_urlfetch_context *ctx = (struct cmd_urlfetch_context *)cmd->context; @@ -72,8 +72,6 @@ struct client *client = cmd->client; struct cmd_urlfetch_context *ctx = (struct cmd_urlfetch_context *)cmd->context; - const unsigned char *data; - size_t size; int ret = 1; /* are we in the middle of an urlfetch literal? */ @@ -88,25 +86,35 @@ } /* transfer literal to client */ - o_stream_set_max_buffer_size(client->output, 4096); - while (i_stream_read_data(ctx->input, &data, &size, 0) > 0) { - if ((ret = o_stream_send(client->output, data, size)) < 0) - break; - i_stream_skip(ctx->input, ret); - } + o_stream_set_max_buffer_size(client->output, 0); + ret = o_stream_send_istream(client->output, ctx->input); o_stream_set_max_buffer_size(client->output, (size_t)-1); - if (ret < 0 || ctx->input->stream_errno != 0) { - /* fatal error */ + if (ctx->input->v_offset == ctx->size) { + /* finished successfully */ + i_stream_unref(&ctx->input); + return 1; + } + if (client->output->closed) { + /* client disconnected */ return -1; } + if (ctx->input->stream_errno != 0) { + errno = ctx->input->stream_errno; + i_error("read(%s) failed: %m (URLFETCH)", + i_stream_get_name(ctx->input)); + client_disconnect(client, "URLFETCH failed"); + return -1; + } + if (!ctx->input->eof) { + o_stream_set_flush_pending(client->output, TRUE); + return 0; + } - if (!ctx->input->eof) - return 0; - - /* finished */ - i_stream_unref(&ctx->input); - return 1; + i_error("URLFETCH got too little data: %"PRIuUOFF_T" vs %"PRIuUOFF_T, + ctx->input->v_offset, ctx->size); + client_disconnect(client, "FETCH failed"); + return -1; } static bool cmd_urlfetch_continue(struct client_command_context *cmd) @@ -208,12 +216,12 @@ if (reply->input != NULL) { ctx->input = reply->input; + ctx->size = reply->size; i_stream_ref(reply->input); ret = cmd_urlfetch_transfer_literal(cmd); if (ret < 0) { ctx->failed = TRUE; - cmd_urlfetch_finish(cmd); return -1; } if (ret == 0) { @@ -240,17 +248,20 @@ struct cmd_urlfetch_context *ctx = cmd->context; int ret; - if (cmd->cancel) - return 1; - if (reply == NULL) { /* fatal failure */ last = TRUE; ctx->failed = TRUE; } else if (reply->succeeded) { /* URL fetch succeeded */ - if ((ret = cmd_urlfetch_url_sucess(cmd, reply)) <= 0) - return ret; + ret = cmd_urlfetch_url_sucess(cmd, reply); + if (ret == 0) + return 0; + if (ret < 0) { + ctx->ufetch = NULL; + cmd_urlfetch_finish(cmd); + return -1; + } } else { /* URL fetch failed */ client_send_line(cmd->client, @@ -262,6 +273,7 @@ } if (last && cmd->state == CLIENT_COMMAND_STATE_WAIT_EXTERNAL) { + ctx->ufetch = NULL; cmd_urlfetch_finish(cmd); client_command_free(&cmd); } diff -r acd76b5272e9 -r 3e70eacf67a4 src/lib-imap-urlauth/imap-urlauth-fetch.c --- a/src/lib-imap-urlauth/imap-urlauth-fetch.c Tue Oct 23 12:59:41 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-fetch.c Tue Oct 23 20:07:06 2012 +0300 @@ -58,23 +58,20 @@ if (ufetch->local_url != NULL) imap_msgpart_url_free(&ufetch->local_url); - if (ufetch->pending_reply.url != NULL) - i_free(ufetch->pending_reply.url); + i_free_and_null(ufetch->pending_reply.url); + i_free_and_null(ufetch->pending_reply.bodypartstruct); + i_free_and_null(ufetch->pending_reply.error); if (ufetch->pending_reply.input != NULL) i_stream_unref(&ufetch->pending_reply.input); - if (ufetch->pending_reply.bodypartstruct != NULL) - i_free(ufetch->pending_reply.bodypartstruct); - if (ufetch->pending_reply.error != NULL) - i_free(ufetch->pending_reply.error); url = ufetch->local_urls_head; - while (url != NULL) { + for (; url != NULL; url = url_next) { url_next = url->next; i_free(url->url); i_free(url); ufetch->pending_requests--; - url = url_next; } + ufetch->local_urls_head = ufetch->local_urls_tail = NULL; } static void imap_urlauth_fetch_abort(struct imap_urlauth_fetch *ufetch) @@ -131,7 +128,7 @@ reply.flags = url_flags; reply.succeeded = FALSE; reply.error = error; - + T_BEGIN { ret = ufetch->callback(&reply, ufetch->pending_requests == 0, ufetch->context); @@ -298,6 +295,8 @@ imap_urlauth_fetch_abort_local(ufetch); ufetch->failed = TRUE; } + if (ret != 0) + imap_urlauth_fetch_deinit(&ufetch); return ret; } diff -r acd76b5272e9 -r 3e70eacf67a4 src/lib-imap-urlauth/imap-urlauth-fetch.h --- a/src/lib-imap-urlauth/imap-urlauth-fetch.h Tue Oct 23 12:59:41 2012 +0300 +++ b/src/lib-imap-urlauth/imap-urlauth-fetch.h Tue Oct 23 20:07:06 2012 +0300 @@ -33,7 +33,8 @@ for next reply, 0 if not all data was processed, and -1 for error. If a callback returns 0, imap_urlauth_connection_continue() must be called once new replies may be processed. If this is the last request to yield a reply, - argument last is TRUE */ + argument last is TRUE. The callback must not call + imap_urlauth_fetch_deinit(). */ typedef int imap_urlauth_fetch_callback_t(struct imap_urlauth_fetch_reply *reply, bool last, void *context); From dovecot at dovecot.org Tue Oct 23 20:16:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:16:21 +0300 Subject: dovecot-2.2: imap: URLFETCH BINARY BODYPARTSTRUCTURE returns bin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3484591230ac changeset: 15237:3484591230ac user: Timo Sirainen date: Tue Oct 23 20:15:36 2012 +0300 description: imap: URLFETCH BINARY BODYPARTSTRUCTURE returns binary-decoded line counts. diffstat: src/lib-imap-storage/imap-msgpart.c | 7 +- src/lib-mail/message-binary-part.c | 7 +- src/lib-mail/message-binary-part.h | 3 + src/lib-storage/fail-mail.c | 1 + src/lib-storage/index/index-mail-binary.c | 91 ++++++++++++++++++++++++++---- src/lib-storage/index/index-mail.h | 3 +- src/lib-storage/mail-storage-private.h | 3 +- src/lib-storage/mail-storage.h | 7 +- src/lib-storage/mail.c | 11 ++- 9 files changed, 107 insertions(+), 26 deletions(-) diffs (truncated from 378 to 300 lines): diff -r d5706259963b -r 3484591230ac src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Tue Oct 23 20:15:36 2012 +0300 @@ -706,6 +706,7 @@ struct imap_msgpart_open_result result; struct message_part *part; bool include_hdr; + unsigned int lines; int ret; if (!msgpart->decode_cte_to_binary || @@ -733,7 +734,7 @@ return -1; } include_hdr = msgpart->fetch_type == FETCH_FULL; - return mail_get_binary_size(mail, part, include_hdr, size_r); + return mail_get_binary_size(mail, part, include_hdr, size_r, &lines); } static int @@ -769,13 +770,15 @@ { struct message_part **pos; uoff_t size; + unsigned int lines; - if (mail_get_binary_size(mail, part, FALSE, &size) < 0) + if (mail_get_binary_size(mail, part, FALSE, &size, &lines) < 0) return -1; *binpart_r = t_new(struct message_part, 1); **binpart_r = *part; (*binpart_r)->body_size.virtual_size = size; + (*binpart_r)->body_size.lines = lines; pos = &(*binpart_r)->children; for (part = part->children; part != NULL; part = part->next) { diff -r d5706259963b -r 3484591230ac src/lib-mail/message-binary-part.c --- a/src/lib-mail/message-binary-part.c Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-mail/message-binary-part.c Tue Oct 23 20:15:36 2012 +0300 @@ -13,6 +13,7 @@ numpack_encode(dest, part->physical_pos); numpack_encode(dest, part->binary_hdr_size); numpack_encode(dest, part->binary_body_size); + numpack_encode(dest, part->binary_body_lines_count); } } @@ -20,7 +21,7 @@ struct message_binary_part **parts_r) { const uint8_t *p = data, *end = p + size; - uint64_t n1, n2, n3; + uint64_t n1, n2, n3, n4; struct message_binary_part *part = NULL, *prev_part = NULL; while (p != end) { @@ -29,11 +30,13 @@ prev_part = part; if (numpack_decode(&p, end, &n1) < 0 || numpack_decode(&p, end, &n2) < 0 || - numpack_decode(&p, end, &n3) < 0) + numpack_decode(&p, end, &n3) < 0 || + numpack_decode(&p, end, &n4) < 0) return -1; part->physical_pos = n1; part->binary_hdr_size = n2; part->binary_body_size = n3; + part->binary_body_lines_count = n4; } *parts_r = part; return 0; diff -r d5706259963b -r 3484591230ac src/lib-mail/message-binary-part.h --- a/src/lib-mail/message-binary-part.h Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-mail/message-binary-part.h Tue Oct 23 20:15:36 2012 +0300 @@ -12,6 +12,9 @@ "binary". */ uoff_t binary_hdr_size; uoff_t binary_body_size; + /* BODYSTRUCTURE for text/ and message/rfc822 parts includes lines + count. Decoding may change these numbers. */ + unsigned int binary_body_lines_count; }; /* Serialize message binary_part. */ diff -r d5706259963b -r 3484591230ac src/lib-storage/fail-mail.c --- a/src/lib-storage/fail-mail.c Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-storage/fail-mail.c Tue Oct 23 20:15:36 2012 +0300 @@ -181,6 +181,7 @@ const struct message_part *part ATTR_UNUSED, bool include_hdr ATTR_UNUSED, uoff_t *size_r ATTR_UNUSED, + unsigned int *body_lines_r ATTR_UNUSED, bool *binary_r ATTR_UNUSED, struct istream **stream_r ATTR_UNUSED) { diff -r d5706259963b -r 3484591230ac src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-storage/index/index-mail-binary.c Tue Oct 23 20:15:36 2012 +0300 @@ -24,7 +24,8 @@ struct binary_block { struct istream *input; - struct message_binary_part bin_part; + uoff_t physical_pos; + unsigned int body_lines_count; bool converted, converted_hdr; }; @@ -130,7 +131,7 @@ if (ctx->copy_start_offset != 0) binary_copy_to(ctx, part->physical_pos); block = array_append_space(&ctx->blocks); - block->bin_part.physical_pos = part->physical_pos; + block->physical_pos = part->physical_pos; block->converted = TRUE; block->converted_hdr = TRUE; @@ -164,7 +165,7 @@ /* single part - write decoded data */ block = array_append_space(&ctx->blocks); - block->bin_part.physical_pos = part->physical_pos; + block->physical_pos = part->physical_pos; i_stream_seek(ctx->input, part->physical_pos + part->header_size.physical_size); @@ -248,7 +249,7 @@ bin_part.physical_pos = part->physical_pos; found = FALSE; for (i = 0; i < count; i++) { - if (blocks[i].bin_part.physical_pos != part->physical_pos || + if (blocks[i].physical_pos != part->physical_pos || !blocks[i].converted) continue; @@ -287,12 +288,62 @@ blocks = array_get(&ctx->blocks, &count); streams = t_new(struct istream *, count+1); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { streams[i] = blocks[i].input; + i_assert(streams[i]->v_offset == 0); + } return streams; } static int +blocks_count_lines(struct binary_ctx *ctx, struct istream *full_input) +{ + struct binary_block *blocks, *cur_block; + unsigned int block_idx, block_count; + uoff_t cur_offset, cur_size; + const unsigned char *data, *p; + size_t size, skip; + ssize_t ret; + + blocks = array_get_modifiable(&ctx->blocks, &block_count); + cur_block = blocks; + cur_offset = 0; + block_idx = 0; + + while ((ret = i_stream_read_data(full_input, &data, &size, 0)) > 0) { + i_assert(cur_offset <= cur_block->input->v_offset); + if (cur_block->input->eof) { + cur_size = cur_block->input->v_offset + + i_stream_get_data_size(cur_block->input); + i_assert(size >= cur_size - cur_offset); + size = cur_size - cur_offset; + } + skip = size; + while ((p = memchr(data, '\n', size)) != NULL) { + size -= p-data+1; + data = p+1; + cur_block->body_lines_count++; + } + i_stream_skip(full_input, skip); + cur_offset += skip; + + if (cur_block->input->eof) { + if (++block_idx == block_count) + cur_block = NULL; + else + cur_block++; + cur_offset = 0; + } + } + i_assert(ret == -1); + if (full_input->stream_errno != 0) + return -1; + i_assert(!i_stream_have_bytes_left(cur_block->input)); + i_assert(block_idx+1 == block_count); + return 0; +} + +static int index_mail_read_binary_to_cache(struct mail *_mail, const struct message_part *part, bool include_hdr, bool *binary_r, @@ -325,8 +376,7 @@ cache->input = i_streams_merge(blocks_get_streams(&ctx), IO_BLOCK_SIZE, fd_callback, _mail); - - if (i_stream_get_size(cache->input, TRUE, &cache->size) < 0) { + if (blocks_count_lines(&ctx, cache->input) < 0) { mail_storage_set_critical(_mail->box->storage, "read(%s) failed: %m", i_stream_get_name(cache->input)); @@ -334,6 +384,9 @@ binary_streams_free(&ctx); return -1; } + i_assert(!i_stream_have_bytes_left(cache->input)); + cache->size = cache->input->v_offset; + i_stream_seek(cache->input, 0); if (part->parent == NULL && include_hdr && mail->data.bin_parts == NULL) { @@ -389,13 +442,14 @@ static int index_mail_get_binary_size(struct mail *_mail, - const struct message_part *part, - bool include_hdr, uoff_t *size_r) + const struct message_part *part, bool include_hdr, + uoff_t *size_r, unsigned int *lines_r) { struct index_mail *mail = (struct index_mail *)_mail; struct message_part *all_parts, *msg_part; const struct message_binary_part *bin_part, *root_bin_part; uoff_t size, end_offset; + unsigned int lines; bool binary, converted; if (mail_get_parts(_mail, &all_parts) < 0) @@ -411,6 +465,11 @@ size = part->header_size.virtual_size + part->body_size.virtual_size; + /* note that we assume here that binary translation doesn't change the + headers' line counts. this isn't true if the original message + contained duplicate Content-Transfer-Encoding lines, but since + that's invalid anyway we don't bother trying to handle it. */ + lines = part->header_size.lines + part->body_size.lines; end_offset = part->physical_pos + size; bin_part = mail->data.bin_parts; root_bin_part = NULL; @@ -428,6 +487,8 @@ msg_part->body_size.virtual_size; size += bin_part->binary_hdr_size + bin_part->binary_body_size; + lines -= msg_part->body_size.lines; + lines += bin_part->binary_body_lines_count; } } if (!include_hdr) { @@ -435,15 +496,18 @@ size -= root_bin_part->binary_hdr_size; else size -= part->header_size.virtual_size; + lines -= part->header_size.lines; } *size_r = size; + *lines_r = lines; return 0; } int index_mail_get_binary_stream(struct mail *_mail, const struct message_part *part, bool include_hdr, uoff_t *size_r, - bool *binary_r, struct istream **stream_r) + unsigned int *lines_r, bool *binary_r, + struct istream **stream_r) { struct index_mail *mail = (struct index_mail *)_mail; struct mail_binary_cache *cache = &_mail->box->storage->binary_cache; @@ -451,9 +515,12 @@ bool binary, converted; if (stream_r == NULL) { - return index_mail_get_binary_size(_mail, part, - include_hdr, size_r); + return index_mail_get_binary_size(_mail, part, include_hdr, + size_r, lines_r); } + /* current implementation doesn't bother implementing this, + because it's not needed by anything. */ + i_assert(lines_r == NULL); /* FIXME: always put the header to temp file. skip it when needed. */ if (cache->box == _mail->box && cache->uid == _mail->uid && diff -r d5706259963b -r 3484591230ac src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Tue Oct 23 20:09:35 2012 +0300 +++ b/src/lib-storage/index/index-mail.h Tue Oct 23 20:15:36 2012 +0300 @@ -207,7 +207,8 @@ int index_mail_get_binary_stream(struct mail *_mail, const struct message_part *part, bool include_hdr, uoff_t *size_r, From dovecot at dovecot.org Tue Oct 23 20:16:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:16:21 +0300 Subject: dovecot-2.2: istream-seekable: When read() reaches EOF, unrefere... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d5706259963b changeset: 15236:d5706259963b user: Timo Sirainen date: Tue Oct 23 20:09:35 2012 +0300 description: istream-seekable: When read() reaches EOF, unreference underlying streams. This was already done when the stream was kept in memory, but not when when the stream was written to temporary file. diffstat: src/lib/istream-seekable.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 3e70eacf67a4 -r d5706259963b src/lib/istream-seekable.c --- a/src/lib/istream-seekable.c Tue Oct 23 20:07:06 2012 +0300 +++ b/src/lib/istream-seekable.c Tue Oct 23 20:09:35 2012 +0300 @@ -153,6 +153,7 @@ /* last one, EOF */ sstream->size = sstream->istream.istream.v_offset; sstream->istream.istream.eof = TRUE; + unref_streams(sstream); return -1; } From dovecot at dovecot.org Tue Oct 23 20:30:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:30:47 +0300 Subject: dovecot-2.2: imap: Another fix for handling partial FETCHes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e9cdf435fde2 changeset: 15238:e9cdf435fde2 user: Timo Sirainen date: Tue Oct 23 20:30:41 2012 +0300 description: imap: Another fix for handling partial FETCHes. diffstat: src/lib-imap-storage/imap-msgpart.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3484591230ac -r e9cdf435fde2 src/lib-imap-storage/imap-msgpart.c --- a/src/lib-imap-storage/imap-msgpart.c Tue Oct 23 20:15:36 2012 +0300 +++ b/src/lib-imap-storage/imap-msgpart.c Tue Oct 23 20:30:41 2012 +0300 @@ -440,7 +440,7 @@ /* update cache */ cache->uid = mail->uid; cache->physical_start = physical_start; - cache->physical_pos = input->v_offset; + cache->physical_pos = input->v_offset - physical_start; cache->virtual_pos = msgpart->partial_offset; if (cr_skipped) { /* the physical_pos points to virtual CRLF, but From dovecot at dovecot.org Tue Oct 23 20:46:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:46:49 +0300 Subject: dovecot-2.2: maildir: Don't assert-crash on mailbox creation. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/488791e8c8d7 changeset: 15239:488791e8c8d7 user: Timo Sirainen date: Tue Oct 23 20:46:12 2012 +0300 description: maildir: Don't assert-crash on mailbox creation. diffstat: src/lib-storage/index/maildir/maildir-storage.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (24 lines): diff -r e9cdf435fde2 -r 488791e8c8d7 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Tue Oct 23 20:30:41 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Tue Oct 23 20:46:12 2012 +0300 @@ -237,14 +237,17 @@ /* create or fix maildir, ignore if it already exists */ static int create_maildir_subdirs(struct mailbox *box, bool verify) { - const char *path; + const char *path, *box_path; unsigned int i; enum mail_error error; int ret = 0; + if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_MAILBOX, + &box_path) < 0) + return -1; + for (i = 0; i < N_ELEMENTS(maildir_subdirs); i++) { - path = t_strconcat(mailbox_get_path(box), "/", - maildir_subdirs[i], NULL); + path = t_strconcat(box_path, "/", maildir_subdirs[i], NULL); if (mkdir_verify(box, path, verify) < 0) { error = mailbox_get_last_mail_error(box); if (error != MAIL_ERROR_EXISTS) From dovecot at dovecot.org Tue Oct 23 20:46:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Oct 2012 20:46:49 +0300 Subject: dovecot-2.2: mailbox_list_index=yes: Fixed refreshing list index... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/92d5b8133bf2 changeset: 15240:92d5b8133bf2 user: Timo Sirainen date: Tue Oct 23 20:46:35 2012 +0300 description: mailbox_list_index=yes: Fixed refreshing list index after mailbox was created. diffstat: src/lib-storage/list/mailbox-list-index.c | 16 ++++++++++++++++ 1 files changed, 16 insertions(+), 0 deletions(-) diffs (33 lines): diff -r 488791e8c8d7 -r 92d5b8133bf2 src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Tue Oct 23 20:46:12 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index.c Tue Oct 23 20:46:35 2012 +0300 @@ -420,6 +420,19 @@ } static int +mailbox_list_index_create_mailbox(struct mailbox *box, + const struct mailbox_update *update, + bool directory) +{ + struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box); + + if (ibox->module_ctx.super.create_box(box, update, directory) < 0) + return -1; + mailbox_list_index_refresh_later(box->list); + return 0; +} + +static int mailbox_list_index_delete_mailbox(struct mailbox_list *list, const char *name) { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); @@ -569,6 +582,9 @@ ibox->module_ctx.super = box->v; MODULE_CONTEXT_SET(box, index_list_storage_module, ibox); + /* for layout=index this gets overridden */ + box->v.create_box = mailbox_list_index_create_mailbox; + mailbox_list_index_status_init_mailbox(box); mailbox_list_index_backend_init_mailbox(box); } From dovecot at dovecot.org Wed Oct 24 09:21:17 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 09:21:17 +0300 Subject: dovecot-2.2: imap: Avoid double-quoting non-atom header field Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e8181fc25500 changeset: 15241:e8181fc25500 user: Timo Sirainen date: Wed Oct 24 09:21:00 2012 +0300 description: imap: Avoid double-quoting non-atom header field diffstat: src/imap/imap-fetch-body.c | 5 +---- 1 files changed, 1 insertions(+), 4 deletions(-) diffs (16 lines): diff -r 92d5b8133bf2 -r e8181fc25500 src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Tue Oct 23 20:46:35 2012 +0300 +++ b/src/imap/imap-fetch-body.c Wed Oct 24 09:21:00 2012 +0300 @@ -221,11 +221,8 @@ if (args[i].type == IMAP_ARG_ATOM) str_append(str, value); - else { - str_append_c(str, '"'); + else imap_dquote_append(str, value); - str_append_c(str, '"'); - } } str_append_c(str, ')'); body->section = str_c(str); From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: lib-imap: Renamed imap_dquote_append() to imap_appe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4137ea599dcd changeset: 15242:4137ea599dcd user: Timo Sirainen date: Wed Oct 24 09:21:46 2012 +0300 description: lib-imap: Renamed imap_dquote_append() to imap_append_quoted() diffstat: src/imap/imap-fetch-body.c | 2 +- src/lib-imap-client/imapc-connection.c | 2 +- src/lib-imap/imap-id.c | 2 +- src/lib-imap/imap-quote.c | 2 +- src/lib-imap/imap-quote.h | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diffs (59 lines): diff -r e8181fc25500 -r 4137ea599dcd src/imap/imap-fetch-body.c --- a/src/imap/imap-fetch-body.c Wed Oct 24 09:21:00 2012 +0300 +++ b/src/imap/imap-fetch-body.c Wed Oct 24 09:21:46 2012 +0300 @@ -222,7 +222,7 @@ if (args[i].type == IMAP_ARG_ATOM) str_append(str, value); else - imap_dquote_append(str, value); + imap_append_quoted(str, value); } str_append_c(str, ')'); body->section = str_c(str); diff -r e8181fc25500 -r 4137ea599dcd src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Wed Oct 24 09:21:00 2012 +0300 +++ b/src/lib-imap-client/imapc-connection.c Wed Oct 24 09:21:46 2012 +0300 @@ -1814,7 +1814,7 @@ const char *arg = va_arg(args, const char *); if (!need_literal(arg)) - imap_dquote_append(cmd->data, arg); + imap_append_quoted(cmd->data, arg); else if ((cmd->conn->capabilities & IMAPC_CAPABILITY_LITERALPLUS) != 0) { str_printfa(cmd->data, "{%"PRIuSIZE_T"+}\r\n%s", diff -r e8181fc25500 -r 4137ea599dcd src/lib-imap/imap-id.c --- a/src/lib-imap/imap-id.c Wed Oct 24 09:21:00 2012 +0300 +++ b/src/lib-imap/imap-id.c Wed Oct 24 09:21:46 2012 +0300 @@ -70,7 +70,7 @@ /* key */ if (str_len(str) > 1) str_append_c(str, ' '); - imap_dquote_append(str, key); + imap_append_quoted(str, key); str_append_c(str, ' '); /* value */ if (IMAP_ARG_IS_EOL(&args[1])) { diff -r e8181fc25500 -r 4137ea599dcd src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Wed Oct 24 09:21:00 2012 +0300 +++ b/src/lib-imap/imap-quote.c Wed Oct 24 09:21:46 2012 +0300 @@ -123,7 +123,7 @@ return ret; } -void imap_dquote_append(string_t *dest, const char *src) +void imap_append_quoted(string_t *dest, const char *src) { str_append_c(dest, '"'); for (; *src != '\0'; src++) { diff -r e8181fc25500 -r 4137ea599dcd src/lib-imap/imap-quote.h --- a/src/lib-imap/imap-quote.h Wed Oct 24 09:21:00 2012 +0300 +++ b/src/lib-imap/imap-quote.h Wed Oct 24 09:21:46 2012 +0300 @@ -16,6 +16,6 @@ size_t value_len, bool fix_text); /* Append data to destination string quoted using "". */ -void imap_dquote_append(string_t *dest, const char *src); +void imap_append_quoted(string_t *dest, const char *src); #endif From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: lib-imap: Added imap_append_string() and imap_appen... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/62d36b2dcbb6 changeset: 15243:62d36b2dcbb6 user: Timo Sirainen date: Wed Oct 24 09:38:06 2012 +0300 description: lib-imap: Added imap_append_string() and imap_append_nstring() diffstat: src/lib-imap/imap-quote.c | 12 ++++++++++++ src/lib-imap/imap-quote.h | 6 +++++- 2 files changed, 17 insertions(+), 1 deletions(-) diffs (38 lines): diff -r 4137ea599dcd -r 62d36b2dcbb6 src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Wed Oct 24 09:21:46 2012 +0300 +++ b/src/lib-imap/imap-quote.c Wed Oct 24 09:38:06 2012 +0300 @@ -123,6 +123,18 @@ return ret; } +void imap_append_string(string_t *dest, const char *src) +{ + i_assert(src != NULL); + + imap_append_nstring(dest, src); +} + +void imap_append_nstring(string_t *dest, const char *src) +{ + imap_quote_append_string(dest, src, FALSE); +} + void imap_append_quoted(string_t *dest, const char *src) { str_append_c(dest, '"'); diff -r 4137ea599dcd -r 62d36b2dcbb6 src/lib-imap/imap-quote.h --- a/src/lib-imap/imap-quote.h Wed Oct 24 09:21:46 2012 +0300 +++ b/src/lib-imap/imap-quote.h Wed Oct 24 09:38:06 2012 +0300 @@ -15,7 +15,11 @@ const char *imap_quote(pool_t pool, const unsigned char *value, size_t value_len, bool fix_text); -/* Append data to destination string quoted using "". */ +/* Append "quoted" or literal. */ +void imap_append_string(string_t *dest, const char *src); +/* Append NIL, "quoted" or literal. */ +void imap_append_nstring(string_t *dest, const char *src); +/* Append "quoted". If src has 8bit chars, skip over them. */ void imap_append_quoted(string_t *dest, const char *src); #endif From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: Use imap_append_*string() instead of imap_quote_app... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff86acd4eef5 changeset: 15244:ff86acd4eef5 user: Timo Sirainen date: Wed Oct 24 09:39:52 2012 +0300 description: Use imap_append_*string() instead of imap_quote_append*() where possible. This makes it clearer what types of output should be sent. diffstat: src/imap-login/imap-proxy.c | 5 ++--- src/imap/cmd-list.c | 11 ++++------- src/imap/cmd-namespace.c | 2 +- src/imap/cmd-notify.c | 2 +- src/imap/imap-fetch.c | 4 ++-- src/imap/imap-notify.c | 4 ++-- src/imap/imap-search.c | 4 ++-- src/imap/imap-status.c | 2 +- src/imap/imap-sync.c | 2 +- src/lib-imap/imap-id.c | 6 +----- src/plugins/imap-acl/imap-acl-plugin.c | 10 +++++----- src/plugins/imap-quota/imap-quota-plugin.c | 6 +++--- 12 files changed, 25 insertions(+), 33 deletions(-) diffs (243 lines): diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap-login/imap-proxy.c --- a/src/imap-login/imap-proxy.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap-login/imap-proxy.c Wed Oct 24 09:39:52 2012 +0300 @@ -75,10 +75,9 @@ if (client->common.proxy_master_user == NULL) { /* logging in normally - use LOGIN command */ str_append(str, "L LOGIN "); - imap_quote_append_string(str, client->common.proxy_user, FALSE); + imap_append_string(str, client->common.proxy_user); str_append_c(str, ' '); - imap_quote_append_string(str, client->common.proxy_password, - FALSE); + imap_append_string(str, client->common.proxy_password); proxy_free_password(&client->common); } else if (client->proxy_sasl_ir) { diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/cmd-list.c --- a/src/imap/cmd-list.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/cmd-list.c Wed Oct 24 09:39:52 2012 +0300 @@ -249,7 +249,7 @@ list_reply_append_ns_sep_param(str, mail_namespace_get_sep(info->ns)); str_append_c(str, ' '); - imap_quote_append_string(str, str_c(mutf7_name), FALSE); + imap_append_string(str, str_c(mutf7_name)); mailbox_childinfo2str(ctx, str, flags); ret = client_send_line_next(ctx->cmd->client, str_c(str)); @@ -333,7 +333,7 @@ str_printfa(str, "%c\" ", ns_sep); if (*ns_prefix != '\0') { /* non-hidden namespace, use it as the root name */ - imap_quote_append_string(str, ns_prefix, FALSE); + imap_append_string(str, ns_prefix); } else { /* Hidden namespace or empty namespace prefix. We could just return an empty root name, but it's safer to emulate what @@ -343,11 +343,8 @@ if (p == NULL) str_append(str, "\"\""); - else { - imap_quote_append_string(str, - t_strdup_until(ref, p + 1), - FALSE); - } + else + imap_append_string(str, t_strdup_until(ref, p + 1)); } client_send_line(client, str_c(str)); } diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/cmd-namespace.c --- a/src/imap/cmd-namespace.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/cmd-namespace.c Wed Oct 24 09:39:52 2012 +0300 @@ -30,7 +30,7 @@ ns->prefix); } - imap_quote_append_string(str, str_c(mutf7_prefix), FALSE); + imap_append_string(str, str_c(mutf7_prefix)); str_append(str, " \""); if (ns_sep == '\\') str_append_c(str, '\\'); diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/cmd-notify.c --- a/src/imap/cmd-notify.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/cmd-notify.c Wed Oct 24 09:39:52 2012 +0300 @@ -365,7 +365,7 @@ str_append_c(str, ns_sep); str_append(str, "\" "); - imap_quote_append_string(str, mailbox_get_vname(box), FALSE); + imap_append_string(str, mailbox_get_vname(box)); client_send_line(client, str_c(str)); } diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/imap-fetch.c --- a/src/imap/imap-fetch.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/imap-fetch.c Wed Oct 24 09:39:52 2012 +0300 @@ -833,7 +833,7 @@ return -1; str_append(ctx->state.cur_str, "X-GUID "); - imap_quote_append_string(ctx->state.cur_str, value, FALSE); + imap_append_string(ctx->state.cur_str, value); str_append_c(ctx->state.cur_str, ' '); return 1; } @@ -860,7 +860,7 @@ i_panic("FETCH: Mailbox name not UTF-8: %s", name); str_append(ctx->state.cur_str, "X-MAILBOX "); - imap_quote_append_string(ctx->state.cur_str, str_c(mutf7_name), FALSE); + imap_append_string(ctx->state.cur_str, str_c(mutf7_name)); str_append_c(ctx->state.cur_str, ' '); return 1; } diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/imap-notify.c --- a/src/imap/imap-notify.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/imap-notify.c Wed Oct 24 09:39:52 2012 +0300 @@ -33,10 +33,10 @@ str_append_c(str, ns_sep); str_append(str, "\" "); - imap_quote_append_string(str, rec->vname, FALSE); + imap_append_string(str, rec->vname); if (rec->old_vname != NULL) { str_append(str, " (\"OLDNAME\" ("); - imap_quote_append_string(str, rec->old_vname, FALSE); + imap_append_string(str, rec->old_vname); str_append(str, "))"); } return client_send_line_next(notify_ns->ctx->client, str_c(str)); diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/imap-search.c --- a/src/imap/imap-search.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/imap-search.c Wed Oct 24 09:39:52 2012 +0300 @@ -178,7 +178,7 @@ /* too many updates */ string_t *str = t_str_new(256); str_append(str, "* NO [NOUPDATE "); - imap_quote_append_string(str, ctx->cmd->tag, FALSE); + imap_append_quoted(str, ctx->cmd->tag); str_append_c(str, ']'); client_send_line(client, str_c(str)); ctx->return_options &= ~SEARCH_RETURN_UPDATE; @@ -318,7 +318,7 @@ str = str_new(default_pool, 1024); str_append(str, "* ESEARCH (TAG "); - imap_quote_append_string(str, ctx->cmd->tag, FALSE); + imap_append_string(str, ctx->cmd->tag); str_append_c(str, ')'); if (ctx->cmd->uid) diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/imap-status.c --- a/src/imap/imap-status.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/imap-status.c Wed Oct 24 09:39:52 2012 +0300 @@ -107,7 +107,7 @@ str = t_str_new(128); str_append(str, "* STATUS "); - imap_quote_append_string(str, mailbox_mutf7, FALSE); + imap_append_string(str, mailbox_mutf7); str_append(str, " ("); prefix_len = str_len(str); diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/imap/imap-sync.c Wed Oct 24 09:39:52 2012 +0300 @@ -152,7 +152,7 @@ cmd = t_str_new(256); str_append(cmd, "* ESEARCH (TAG "); - imap_quote_append_string(cmd, update->tag, FALSE); + imap_append_string(cmd, update->tag); str_append_c(cmd, ')'); if (update->return_uids) str_append(cmd, " UID"); diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/lib-imap/imap-id.c --- a/src/lib-imap/imap-id.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/lib-imap/imap-id.c Wed Oct 24 09:39:52 2012 +0300 @@ -84,11 +84,7 @@ if (strcmp(value, "*") == 0) value = imap_id_get_default(key); } - - if (value == NULL) - str_append(str, "NIL"); - else - imap_quote_append_string(str, value, FALSE); + imap_append_nstring(str, value); } } if (str_len(str) == 1) { diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/plugins/imap-acl/imap-acl-plugin.c --- a/src/plugins/imap-acl/imap-acl-plugin.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/plugins/imap-acl/imap-acl-plugin.c Wed Oct 24 09:39:52 2012 +0300 @@ -163,7 +163,7 @@ i_unreached(); } - imap_quote_append(dest, str_data(tmp), str_len(tmp), FALSE); + imap_append_string(dest, str_c(tmp)); str_append_c(dest, ' '); imap_acl_write_rights_list(dest, rights); } @@ -288,7 +288,7 @@ str = t_str_new(128); str_append(str, "* ACL "); - imap_quote_append_string(str, mailbox, FALSE); + imap_append_string(str, mailbox); ns = mailbox_get_namespace(box); backend = acl_mailbox_list_get_backend(ns->list); @@ -347,7 +347,7 @@ str = t_str_new(128); str_append(str, "* MYRIGHTS "); - imap_quote_append_string(str, orig_mailbox, FALSE); + imap_append_string(str, orig_mailbox); str_append_c(str,' '); imap_acl_write_rights_list(str, rights); @@ -372,9 +372,9 @@ str = t_str_new(128); str_append(str, "* LISTRIGHTS "); - imap_quote_append_string(str, mailbox, FALSE); + imap_append_string(str, mailbox); str_append_c(str, ' '); - imap_quote_append_string(str, identifier, FALSE); + imap_append_string(str, identifier); str_append_c(str, ' '); str_append(str, "\"\" l r w s t p i e k x a c d"); diff -r 62d36b2dcbb6 -r ff86acd4eef5 src/plugins/imap-quota/imap-quota-plugin.c --- a/src/plugins/imap-quota/imap-quota-plugin.c Wed Oct 24 09:38:06 2012 +0300 +++ b/src/plugins/imap-quota/imap-quota-plugin.c Wed Oct 24 09:39:52 2012 +0300 @@ -43,7 +43,7 @@ str_append(str, "* QUOTA "); name = imap_quota_root_get_name(user, owner, root); - imap_quote_append_string(str, name, FALSE); + imap_append_string(str, name); str_append(str, " ("); list = quota_root_get_resources(root); @@ -102,13 +102,13 @@ quotaroot_reply = t_str_new(128); quota_reply = t_str_new(256); str_append(quotaroot_reply, "* QUOTAROOT "); - imap_quote_append_string(quotaroot_reply, orig_mailbox, FALSE); + imap_append_string(quotaroot_reply, orig_mailbox); iter = quota_root_iter_init(box); while ((root = quota_root_iter_next(iter)) != NULL) { str_append_c(quotaroot_reply, ' '); name = imap_quota_root_get_name(client->user, ns->owner, root); - imap_quote_append_string(quotaroot_reply, name, FALSE); + imap_append_string(quotaroot_reply, name); quota_reply_write(quota_reply, client->user, ns->owner, root); } From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: lib-imap: Changed public IS_ATOM*() macros to match... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/42f99a4fc763 changeset: 15245:42f99a4fc763 user: Timo Sirainen date: Wed Oct 24 10:05:37 2012 +0300 description: lib-imap: Changed public IS_ATOM*() macros to match RFC 3501 exactly. Move the imap-parser specific "atom plus some more" parsing macro inside imap-parser.c so it's not used by anyone else. diffstat: src/lib-imap/imap-arg.h | 33 +++++++++++++++++++++++---------- src/lib-imap/imap-parser.c | 12 +++++++++++- src/lib-storage/mailbox-keywords.c | 11 +++++------ 3 files changed, 39 insertions(+), 17 deletions(-) diffs (96 lines): diff -r ff86acd4eef5 -r 42f99a4fc763 src/lib-imap/imap-arg.h --- a/src/lib-imap/imap-arg.h Wed Oct 24 09:39:52 2012 +0300 +++ b/src/lib-imap/imap-arg.h Wed Oct 24 10:05:37 2012 +0300 @@ -3,19 +3,32 @@ #include "array.h" -/* We use this macro to read atoms from input. It should probably contain - everything some day, but for now we can't handle some input otherwise: +/* ABNF: - ']' is required for parsing section (FETCH BODY[]) - '%', '*' and ']' are valid list-chars for LIST patterns - '\' is used in flags */ -#define IS_ATOM_SPECIAL_INPUT(c) \ - ((c) == '(' || (c) == ')' || (c) == '{' || \ - (c) == '"' || (c) <= 32 || (c) == 0x7f) + CHAR = %x01-7F + CTL = %x00-1F / %x7F + SP = %x20 + DQUOTE = %x22 */ +/* ASTRING-CHAR = ATOM-CHAR / resp-specials */ +#define IS_ASTRING_CHAR(c) (IS_ATOM_CHAR(c) || IS_RESP_SPECIAL(c)) +/* ATOM-CHAR = */ +#define IS_ATOM_CHAR(c) (!IS_ATOM_SPECIAL(c)) +/* atom-specials = "(" / ")" / "{" / SP / CTL / list-wildcards / + quoted-specials / resp-specials + Since atoms are only 7bit, we'll also optimize a bit by assuming 8bit chars + are also atom-specials. */ #define IS_ATOM_SPECIAL(c) \ - (IS_ATOM_SPECIAL_INPUT(c) || \ - (c) == ']' || (c) == '%' || (c) == '*' || (c) == '\\') + ((unsigned char)(c) <= 0x20 || (unsigned char)(c) >= 0x7f || \ + (c) == '(' || (c) == ')' || (c) == '{' || IS_LIST_WILDCARD(c) || \ + IS_QUOTED_SPECIAL(c) || IS_RESP_SPECIAL(c)) + +/* list-wildcards = "%" / "*" */ +#define IS_LIST_WILDCARD(c) ((c) == '%' || (c) == '*') +/* quoted-specials = DQUOTE / "\" */ +#define IS_QUOTED_SPECIAL(c) ((c) == '\"' || (c) == '\\') +/* resp-specials = "]" */ +#define IS_RESP_SPECIAL(c) ((c) == ']') enum imap_arg_type { IMAP_ARG_NIL = 0, diff -r ff86acd4eef5 -r 42f99a4fc763 src/lib-imap/imap-parser.c --- a/src/lib-imap/imap-parser.c Wed Oct 24 09:39:52 2012 +0300 +++ b/src/lib-imap/imap-parser.c Wed Oct 24 10:05:37 2012 +0300 @@ -6,6 +6,16 @@ #include "strescape.h" #include "imap-parser.h" +/* We use this macro to read atoms from input. It should probably contain + everything some day, but for now we can't handle some input otherwise: + + ']' is required for parsing section (FETCH BODY[]) + '%', '*' and ']' are valid list-chars for LIST patterns + '\' is used in flags */ +#define IS_ATOM_PARSER_INPUT(c) \ + ((c) == '(' || (c) == ')' || (c) == '{' || \ + (c) == '"' || (c) <= 32 || (c) == 0x7f) + #define is_linebreak(c) \ ((c) == '\r' || (c) == '\n') @@ -280,7 +290,7 @@ { const char *error; - if (IS_ATOM_SPECIAL_INPUT((unsigned char)chr)) + if (IS_ATOM_PARSER_INPUT((unsigned char)chr)) error = "Invalid characters in atom"; else if ((chr & 0x80) != 0) error = "8bit data in atom"; diff -r ff86acd4eef5 -r 42f99a4fc763 src/lib-storage/mailbox-keywords.c --- a/src/lib-storage/mailbox-keywords.c Wed Oct 24 09:39:52 2012 +0300 +++ b/src/lib-storage/mailbox-keywords.c Wed Oct 24 10:05:37 2012 +0300 @@ -117,12 +117,11 @@ /* these are IMAP-specific restrictions, but for now IMAP is all we care about */ for (i = 0; keyword[i] != '\0'; i++) { - if (IS_ATOM_SPECIAL((unsigned char)keyword[i])) { - *error_r = "Invalid characters in keyword"; - return FALSE; - } - if ((unsigned char)keyword[i] >= 0x80) { - *error_r = "8bit characters in keyword"; + if (!IS_ATOM_CHAR(keyword[i])) { + if ((unsigned char)keyword[i] < 0x80) + *error_r = "Invalid characters in keyword"; + else + *error_r = "8bit characters in keyword"; return FALSE; } } From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: lib-imap: Added imap_append_astring() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d9c44aafd163 changeset: 15246:d9c44aafd163 user: Timo Sirainen date: Wed Oct 24 10:08:06 2012 +0300 description: lib-imap: Added imap_append_astring() diffstat: src/lib-imap/imap-quote.c | 19 +++++++++++++++++++ src/lib-imap/imap-quote.h | 2 ++ 2 files changed, 21 insertions(+), 0 deletions(-) diffs (48 lines): diff -r 42f99a4fc763 -r d9c44aafd163 src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Wed Oct 24 10:05:37 2012 +0300 +++ b/src/lib-imap/imap-quote.c Wed Oct 24 10:08:06 2012 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "imap-arg.h" #include "imap-quote.h" void imap_quote_append(string_t *str, const unsigned char *value, @@ -130,6 +131,24 @@ imap_append_nstring(dest, src); } +void imap_append_astring(string_t *dest, const char *src) +{ + unsigned int i; + + i_assert(src != NULL); + + for (i = 0; src[i] != '\0'; i++) { + if (!IS_ASTRING_CHAR(src[i])) { + imap_append_string(dest, src); + return; + } + } + if (i == 0) + imap_append_string(dest, src); + else + str_append(dest, src); +} + void imap_append_nstring(string_t *dest, const char *src) { imap_quote_append_string(dest, src, FALSE); diff -r 42f99a4fc763 -r d9c44aafd163 src/lib-imap/imap-quote.h --- a/src/lib-imap/imap-quote.h Wed Oct 24 10:05:37 2012 +0300 +++ b/src/lib-imap/imap-quote.h Wed Oct 24 10:08:06 2012 +0300 @@ -17,6 +17,8 @@ /* Append "quoted" or literal. */ void imap_append_string(string_t *dest, const char *src); +/* Append atom, "quoted" or literal. */ +void imap_append_astring(string_t *dest, const char *src); /* Append NIL, "quoted" or literal. */ void imap_append_nstring(string_t *dest, const char *src); /* Append "quoted". If src has 8bit chars, skip over them. */ From dovecot at dovecot.org Wed Oct 24 10:14:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 10:14:29 +0300 Subject: dovecot-2.2: Use imap_append_astring() instead of imap_append_st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9a17faaed2eb changeset: 15247:9a17faaed2eb user: Timo Sirainen date: Wed Oct 24 10:14:17 2012 +0300 description: Use imap_append_astring() instead of imap_append_string() where possible. diffstat: src/imap/cmd-list.c | 6 +++--- src/imap/cmd-notify.c | 2 +- src/imap/imap-fetch.c | 4 ++-- src/imap/imap-notify.c | 4 ++-- src/imap/imap-status.c | 2 +- src/plugins/imap-acl/imap-acl-plugin.c | 10 +++++----- src/plugins/imap-quota/imap-quota-plugin.c | 6 +++--- 7 files changed, 17 insertions(+), 17 deletions(-) diffs (161 lines): diff -r d9c44aafd163 -r 9a17faaed2eb src/imap/cmd-list.c --- a/src/imap/cmd-list.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/imap/cmd-list.c Wed Oct 24 10:14:17 2012 +0300 @@ -249,7 +249,7 @@ list_reply_append_ns_sep_param(str, mail_namespace_get_sep(info->ns)); str_append_c(str, ' '); - imap_append_string(str, str_c(mutf7_name)); + imap_append_astring(str, str_c(mutf7_name)); mailbox_childinfo2str(ctx, str, flags); ret = client_send_line_next(ctx->cmd->client, str_c(str)); @@ -333,7 +333,7 @@ str_printfa(str, "%c\" ", ns_sep); if (*ns_prefix != '\0') { /* non-hidden namespace, use it as the root name */ - imap_append_string(str, ns_prefix); + imap_append_astring(str, ns_prefix); } else { /* Hidden namespace or empty namespace prefix. We could just return an empty root name, but it's safer to emulate what @@ -344,7 +344,7 @@ if (p == NULL) str_append(str, "\"\""); else - imap_append_string(str, t_strdup_until(ref, p + 1)); + imap_append_astring(str, t_strdup_until(ref, p + 1)); } client_send_line(client, str_c(str)); } diff -r d9c44aafd163 -r 9a17faaed2eb src/imap/cmd-notify.c --- a/src/imap/cmd-notify.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/imap/cmd-notify.c Wed Oct 24 10:14:17 2012 +0300 @@ -365,7 +365,7 @@ str_append_c(str, ns_sep); str_append(str, "\" "); - imap_append_string(str, mailbox_get_vname(box)); + imap_append_astring(str, mailbox_get_vname(box)); client_send_line(client, str_c(str)); } diff -r d9c44aafd163 -r 9a17faaed2eb src/imap/imap-fetch.c --- a/src/imap/imap-fetch.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/imap/imap-fetch.c Wed Oct 24 10:14:17 2012 +0300 @@ -833,7 +833,7 @@ return -1; str_append(ctx->state.cur_str, "X-GUID "); - imap_append_string(ctx->state.cur_str, value); + imap_append_astring(ctx->state.cur_str, value); str_append_c(ctx->state.cur_str, ' '); return 1; } @@ -860,7 +860,7 @@ i_panic("FETCH: Mailbox name not UTF-8: %s", name); str_append(ctx->state.cur_str, "X-MAILBOX "); - imap_append_string(ctx->state.cur_str, str_c(mutf7_name)); + imap_append_astring(ctx->state.cur_str, str_c(mutf7_name)); str_append_c(ctx->state.cur_str, ' '); return 1; } diff -r d9c44aafd163 -r 9a17faaed2eb src/imap/imap-notify.c --- a/src/imap/imap-notify.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/imap/imap-notify.c Wed Oct 24 10:14:17 2012 +0300 @@ -33,10 +33,10 @@ str_append_c(str, ns_sep); str_append(str, "\" "); - imap_append_string(str, rec->vname); + imap_append_astring(str, rec->vname); if (rec->old_vname != NULL) { str_append(str, " (\"OLDNAME\" ("); - imap_append_string(str, rec->old_vname); + imap_append_astring(str, rec->old_vname); str_append(str, "))"); } return client_send_line_next(notify_ns->ctx->client, str_c(str)); diff -r d9c44aafd163 -r 9a17faaed2eb src/imap/imap-status.c --- a/src/imap/imap-status.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/imap/imap-status.c Wed Oct 24 10:14:17 2012 +0300 @@ -107,7 +107,7 @@ str = t_str_new(128); str_append(str, "* STATUS "); - imap_append_string(str, mailbox_mutf7); + imap_append_astring(str, mailbox_mutf7); str_append(str, " ("); prefix_len = str_len(str); diff -r d9c44aafd163 -r 9a17faaed2eb src/plugins/imap-acl/imap-acl-plugin.c --- a/src/plugins/imap-acl/imap-acl-plugin.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/plugins/imap-acl/imap-acl-plugin.c Wed Oct 24 10:14:17 2012 +0300 @@ -163,7 +163,7 @@ i_unreached(); } - imap_append_string(dest, str_c(tmp)); + imap_append_astring(dest, str_c(tmp)); str_append_c(dest, ' '); imap_acl_write_rights_list(dest, rights); } @@ -288,7 +288,7 @@ str = t_str_new(128); str_append(str, "* ACL "); - imap_append_string(str, mailbox); + imap_append_astring(str, mailbox); ns = mailbox_get_namespace(box); backend = acl_mailbox_list_get_backend(ns->list); @@ -347,7 +347,7 @@ str = t_str_new(128); str_append(str, "* MYRIGHTS "); - imap_append_string(str, orig_mailbox); + imap_append_astring(str, orig_mailbox); str_append_c(str,' '); imap_acl_write_rights_list(str, rights); @@ -372,9 +372,9 @@ str = t_str_new(128); str_append(str, "* LISTRIGHTS "); - imap_append_string(str, mailbox); + imap_append_astring(str, mailbox); str_append_c(str, ' '); - imap_append_string(str, identifier); + imap_append_astring(str, identifier); str_append_c(str, ' '); str_append(str, "\"\" l r w s t p i e k x a c d"); diff -r d9c44aafd163 -r 9a17faaed2eb src/plugins/imap-quota/imap-quota-plugin.c --- a/src/plugins/imap-quota/imap-quota-plugin.c Wed Oct 24 10:08:06 2012 +0300 +++ b/src/plugins/imap-quota/imap-quota-plugin.c Wed Oct 24 10:14:17 2012 +0300 @@ -43,7 +43,7 @@ str_append(str, "* QUOTA "); name = imap_quota_root_get_name(user, owner, root); - imap_append_string(str, name); + imap_append_astring(str, name); str_append(str, " ("); list = quota_root_get_resources(root); @@ -102,13 +102,13 @@ quotaroot_reply = t_str_new(128); quota_reply = t_str_new(256); str_append(quotaroot_reply, "* QUOTAROOT "); - imap_append_string(quotaroot_reply, orig_mailbox); + imap_append_astring(quotaroot_reply, orig_mailbox); iter = quota_root_iter_init(box); while ((root = quota_root_iter_next(iter)) != NULL) { str_append_c(quotaroot_reply, ' '); name = imap_quota_root_get_name(client->user, ns->owner, root); - imap_append_string(quotaroot_reply, name); + imap_append_astring(quotaroot_reply, name); quota_reply_write(quota_reply, client->user, ns->owner, root); } From dovecot at dovecot.org Wed Oct 24 11:22:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 11:22:52 +0300 Subject: dovecot-2.2: lib-imap: bodystructure parsing now uses imap_appen... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cb143b430787 changeset: 15248:cb143b430787 user: Timo Sirainen date: Wed Oct 24 11:22:28 2012 +0300 description: lib-imap: bodystructure parsing now uses imap_append_string() instead of imap_quote*() Some of the fields are parsed through rfc822_*() which guarantees that they contain proper clean input. Other fields are also machine-readable and don't benefit from having whitespace compressed or of any other things that imap_quote*(fix_text=TRUE) did. None of the fields in diffstat: src/lib-imap/imap-bodystructure.c | 68 ++++++++++++++++++-------------------- 1 files changed, 33 insertions(+), 35 deletions(-) diffs (152 lines): diff -r 9a17faaed2eb -r cb143b430787 src/lib-imap/imap-bodystructure.c --- a/src/lib-imap/imap-bodystructure.c Wed Oct 24 10:14:17 2012 +0300 +++ b/src/lib-imap/imap-bodystructure.c Wed Oct 24 11:22:28 2012 +0300 @@ -20,6 +20,14 @@ #define NVL(str, nullstr) ((str) != NULL ? (str) : (nullstr)) +static char *imap_get_string(pool_t pool, const char *value) +{ + string_t *str = t_str_new(64); + + imap_append_string(str, value); + return p_strdup(pool, str_c(str)); +} + static void parse_content_type(struct message_part_body_data *data, struct message_header_line *hdr) { @@ -41,12 +49,12 @@ for (i = 0; value[i] != '\0'; i++) { if (value[i] == '/') { data->content_subtype = - imap_quote(data->pool, str_data(str) + i + 1, - str_len(str) - (i + 1), TRUE); + imap_get_string(data->pool, value + i+1); break; } } - data->content_type = imap_quote(data->pool, str_data(str), i, TRUE); + str_truncate(str, i); + data->content_type = imap_get_string(data->pool, str_c(str)); /* parse parameters and save them */ str_truncate(str, 0); @@ -56,9 +64,9 @@ charset_found = TRUE; str_append_c(str, ' '); - imap_quote_append_string(str, results[0], TRUE); + imap_append_string(str, results[0]); str_append_c(str, ' '); - imap_quote_append_string(str, results[1], TRUE); + imap_append_string(str, results[1]); } if (!charset_found && @@ -85,8 +93,7 @@ str = t_str_new(256); if (rfc822_parse_mime_token(&parser, str) >= 0) { data->content_transfer_encoding = - imap_quote(data->pool, str_data(str), - str_len(str), TRUE); + imap_get_string(data->pool, str_c(str)); } } @@ -103,17 +110,16 @@ str = t_str_new(256); if (rfc822_parse_mime_token(&parser, str) < 0) return; - data->content_disposition = - imap_quote(data->pool, str_data(str), str_len(str), TRUE); + data->content_disposition = imap_get_string(data->pool, str_c(str)); /* parse parameters and save them */ str_truncate(str, 0); rfc2231_parse(&parser, &results); for (; *results != NULL; results += 2) { str_append_c(str, ' '); - imap_quote_append_string(str, results[0], TRUE); + imap_append_string(str, results[0]); str_append_c(str, ' '); - imap_quote_append_string(str, results[1], TRUE); + imap_append_string(str, results[1]); } if (str_len(str) > 0) { data->content_disposition_params = @@ -158,32 +164,26 @@ pool_t pool) { const char *name = hdr->name + strlen("Content-"); - const unsigned char *value; - size_t value_len; + const char *value; if (hdr->continues) { hdr->use_full_value = TRUE; return; } - value = hdr->full_value; - value_len = hdr->full_value_len; + value = t_strndup(hdr->full_value, hdr->full_value_len); switch (*name) { case 'i': case 'I': - if (strcasecmp(name, "ID") == 0 && d->content_id == NULL) { - d->content_id = - imap_quote(pool, value, value_len, TRUE); - } + if (strcasecmp(name, "ID") == 0 && d->content_id == NULL) + d->content_id = imap_get_string(pool, value); break; case 'm': case 'M': - if (strcasecmp(name, "MD5") == 0 && d->content_md5 == NULL) { - d->content_md5 = - imap_quote(pool, value, value_len, TRUE); - } + if (strcasecmp(name, "MD5") == 0 && d->content_md5 == NULL) + d->content_md5 = imap_get_string(pool, value); break; case 't': @@ -198,25 +198,23 @@ case 'l': case 'L': if (strcasecmp(name, "Language") == 0 && - d->content_language == NULL) - parse_content_language(value, value_len, d); - else if (strcasecmp(name, "Location") == 0 && - d->content_location == NULL) { - d->content_location = - imap_quote(pool, value, value_len, TRUE); + d->content_language == NULL) { + parse_content_language(hdr->full_value, + hdr->full_value_len, d); + } else if (strcasecmp(name, "Location") == 0 && + d->content_location == NULL) { + d->content_location = imap_get_string(pool, value); } break; case 'd': case 'D': if (strcasecmp(name, "Description") == 0 && - d->content_description == NULL) { - d->content_description = - imap_quote(pool, value, value_len, TRUE); - } else if (strcasecmp(name, "Disposition") == 0 && - d->content_disposition_params == NULL) { + d->content_description == NULL) + d->content_description = imap_get_string(pool, value); + else if (strcasecmp(name, "Disposition") == 0 && + d->content_disposition_params == NULL) parse_content_disposition(d, hdr); - } break; } } From dovecot at dovecot.org Wed Oct 24 11:30:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 11:30:22 +0300 Subject: dovecot-2.2: lib-imap: ENVELOPE parsing/writing no longer strips... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c849ae6603f2 changeset: 15249:c849ae6603f2 user: Timo Sirainen date: Wed Oct 24 11:28:38 2012 +0300 description: lib-imap: ENVELOPE parsing/writing no longer strips whitespace from machine readable fields. diffstat: src/lib-imap/imap-envelope.c | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-) diffs (40 lines): diff -r cb143b430787 -r c849ae6603f2 src/lib-imap/imap-envelope.c --- a/src/lib-imap/imap-envelope.c Wed Oct 24 11:22:28 2012 +0300 +++ b/src/lib-imap/imap-envelope.c Wed Oct 24 11:28:38 2012 +0300 @@ -147,11 +147,17 @@ *addr_p = message_address_parse(pool, hdr->full_value, hdr->full_value_len, (unsigned int)-1, TRUE); - } + } else if (str_p != NULL) { + if (str_p != &d->subject) { + string_t *str = t_str_new(128); - if (str_p != NULL) { - *str_p = imap_quote(pool, hdr->full_value, - hdr->full_value_len, TRUE); + imap_append_string(str, + t_strndup(hdr->full_value, hdr->full_value_len)); + *str_p = p_strdup(pool, str_c(str)); + } else { + *str_p = imap_quote(pool, hdr->full_value, + hdr->full_value_len, TRUE); + } } } @@ -167,11 +173,11 @@ str_append_c(str, '('); imap_quote_append_string(str, addr->name, TRUE); str_append_c(str, ' '); - imap_quote_append_string(str, addr->route, TRUE); + imap_append_nstring(str, addr->route); str_append_c(str, ' '); - imap_quote_append_string(str, addr->mailbox, TRUE); + imap_append_nstring(str, addr->mailbox); str_append_c(str, ' '); - imap_quote_append_string(str, addr->domain, TRUE); + imap_append_nstring(str, addr->domain); str_append_c(str, ')'); addr = addr->next; From dovecot at dovecot.org Wed Oct 24 11:30:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 11:30:22 +0300 Subject: dovecot-2.2: lib-imap: Replaced last traces of imap_quote*() wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b89dae6aead4 changeset: 15250:b89dae6aead4 user: Timo Sirainen date: Wed Oct 24 11:30:09 2012 +0300 description: lib-imap: Replaced last traces of imap_quote*() with imap_append_string_for_humans() Also the imap_append_*string() functions now decide whether to use quoted-string or literal based on the output string length (if both would work). diffstat: src/lib-imap/imap-envelope.c | 21 ++- src/lib-imap/imap-quote.c | 260 ++++++++++++++++++++++-------------------- src/lib-imap/imap-quote.h | 20 +-- 3 files changed, 158 insertions(+), 143 deletions(-) diffs (truncated from 368 to 300 lines): diff -r c849ae6603f2 -r b89dae6aead4 src/lib-imap/imap-envelope.c --- a/src/lib-imap/imap-envelope.c Wed Oct 24 11:28:38 2012 +0300 +++ b/src/lib-imap/imap-envelope.c Wed Oct 24 11:30:09 2012 +0300 @@ -147,18 +147,18 @@ *addr_p = message_address_parse(pool, hdr->full_value, hdr->full_value_len, (unsigned int)-1, TRUE); - } else if (str_p != NULL) { + } else if (str_p != NULL) T_BEGIN { + string_t *str = t_str_new(128); + if (str_p != &d->subject) { - string_t *str = t_str_new(128); - imap_append_string(str, t_strndup(hdr->full_value, hdr->full_value_len)); - *str_p = p_strdup(pool, str_c(str)); } else { - *str_p = imap_quote(pool, hdr->full_value, - hdr->full_value_len, TRUE); + imap_append_string_for_humans(str, + hdr->full_value, hdr->full_value_len); } - } + *str_p = p_strdup(pool, str_c(str)); + } T_END; } static void imap_write_address(string_t *str, struct message_address *addr) @@ -171,7 +171,12 @@ str_append_c(str, '('); while (addr != NULL) { str_append_c(str, '('); - imap_quote_append_string(str, addr->name, TRUE); + if (addr->name == NULL) + str_append(str, "NIL"); + else { + imap_append_string_for_humans(str, + (const void *)addr->name, strlen(addr->name)); + } str_append_c(str, ' '); imap_append_nstring(str, addr->route); str_append_c(str, ' '); diff -r c849ae6603f2 -r b89dae6aead4 src/lib-imap/imap-quote.c --- a/src/lib-imap/imap-quote.c Wed Oct 24 11:28:38 2012 +0300 +++ b/src/lib-imap/imap-quote.c Wed Oct 24 11:30:09 2012 +0300 @@ -5,124 +5,15 @@ #include "imap-arg.h" #include "imap-quote.h" -void imap_quote_append(string_t *str, const unsigned char *value, - size_t value_len, bool fix_text) -{ - size_t i, extra = 0; - bool last_lwsp = TRUE, literal = FALSE, modify = FALSE; - - if (value == NULL) { - str_append(str, "NIL"); - return; - } - - if (value_len == (size_t)-1) - value_len = strlen((const char *) value); - - for (i = 0; i < value_len; i++) { - switch (value[i]) { - case 0: - /* it's converted to 8bit char */ - literal = TRUE; - last_lwsp = FALSE; - modify = TRUE; - break; - case '\t': - modify = TRUE; - /* fall through */ - case ' ': - if (last_lwsp && fix_text) { - modify = TRUE; - extra++; - } - last_lwsp = TRUE; - break; - case 13: - case 10: - if (!fix_text) - literal = TRUE; - extra++; - modify = TRUE; - break; - default: - if ((value[i] & 0x80) != 0 || - value[i] == '"' || value[i] == '\\') - literal = TRUE; - last_lwsp = FALSE; - } - } - - if (!fix_text) { - extra = 0; - modify = FALSE; - } - - if (!literal) { - /* no 8bit chars or imapspecials, return as "string" */ - str_append_c(str, '"'); - } else { - /* return as literal */ - str_printfa(str, "{%"PRIuSIZE_T"}\r\n", value_len - extra); - } - - if (!modify) - str_append_n(str, value, value_len); - else { - last_lwsp = TRUE; - for (i = 0; i < value_len; i++) { - switch (value[i]) { - case 0: - str_append_c(str, 128); - last_lwsp = FALSE; - break; - case ' ': - case '\t': - if (!last_lwsp) - str_append_c(str, ' '); - last_lwsp = TRUE; - break; - case 13: - case 10: - break; - default: - last_lwsp = FALSE; - str_append_c(str, value[i]); - break; - } - } - } - - if (!literal) - str_append_c(str, '"'); -} - -static const char * -imap_quote_internal(pool_t pool, const unsigned char *value, - size_t value_len, bool fix_text) -{ - string_t *str; - - str = t_str_new(value_len + MAX_INT_STRLEN + 5); - imap_quote_append(str, value, value_len, fix_text); - return pool->datastack_pool ? str_c(str) : - p_strndup(pool, str_data(str), str_len(str)); -} - -const char *imap_quote(pool_t pool, const unsigned char *value, - size_t value_len, bool fix_text) -{ - const char *ret; - - if (value == NULL) - return "NIL"; - - if (pool->datastack_pool) - ret = imap_quote_internal(pool, value, value_len, fix_text); - else T_BEGIN { - ret = imap_quote_internal(pool, value, value_len, fix_text); - } T_END; - return ret; -} +/* If we have quoted-specials (<">, <\>) in a string, the minimum quoted-string + overhead is 3 bytes ("\") while the minimum literal overhead is 5 bytes + ("{n}\r\n"). But the literal overhead also depends on the string size. If + the string length is less than 10, literal catches up to quoted-string after + 3 quoted-specials. If the string length is 10..99, it catches up after 4 + quoted-specials, and so on. We'll assume that the string lengths are usually + in double digits, so we'll switch to literals after seeing 4 + quoted-specials. */ +#define QUOTED_MAX_ESCAPE_CHARS 4 void imap_append_string(string_t *dest, const char *src) { @@ -149,9 +40,52 @@ str_append(dest, src); } +static void +imap_append_literal(string_t *dest, const char *src, unsigned int pos) +{ + unsigned int full_len = pos + strlen(src+pos); + + str_printfa(dest, "{%u}\r\n", full_len); + buffer_append(dest, src, full_len); +} + void imap_append_nstring(string_t *dest, const char *src) { - imap_quote_append_string(dest, src, FALSE); + unsigned int i, escape_count = 0; + + if (src == NULL) { + str_append(dest, "NIL"); + return; + } + + /* first check if we can (or want to) write this as quoted or + as literal. + + quoted-specials = DQUOTE / "\" + QUOTED-CHAR = / + "\" quoted-specials + TEXT-CHAR = + */ + for (i = 0; src[i] != '\0'; i++) { + switch (src[i]) { + case '"': + case '\\': + if (escape_count++ < QUOTED_MAX_ESCAPE_CHARS) + break; + /* fall through */ + case 13: + case 10: + imap_append_literal(dest, src, i); + return; + default: + if ((unsigned char)src[i] >= 0x80) { + imap_append_literal(dest, src, i); + return; + } + break; + } + } + imap_append_quoted(dest, src); } void imap_append_quoted(string_t *dest, const char *src) @@ -159,8 +93,8 @@ str_append_c(dest, '"'); for (; *src != '\0'; src++) { switch (*src) { - case '\r': - case '\n': + case 13: + case 10: /* not allowed */ break; case '"': @@ -180,3 +114,87 @@ } str_append_c(dest, '"'); } + +void imap_append_string_for_humans(string_t *dest, + const unsigned char *src, size_t size) +{ + size_t i, pos, remove_count = 0; + bool last_lwsp = TRUE, modify = FALSE; + + /* first check if there is anything to change */ + for (i = 0; i < size; i++) { + switch (src[i]) { + case 0: + /* convert NUL to #0x80 */ + last_lwsp = FALSE; + modify = TRUE; + break; + case '\t': + modify = TRUE; + /* fall through */ + case ' ': + if (last_lwsp) { + modify = TRUE; + remove_count++; + } + last_lwsp = TRUE; + break; + case 13: + case 10: + remove_count++; + modify = TRUE; + break; + case '"': + case '\\': + modify = TRUE; + last_lwsp = FALSE; + break; + default: + if ((src[i] & 0x80) != 0) + modify = TRUE; + last_lwsp = FALSE; + break; + } + } + if (last_lwsp) { + modify = TRUE; + remove_count++; + } + if (!modify) { + /* fast path: we can simply write it as quoted string + without any escaping */ + str_append_c(dest, '"'); + str_append_n(dest, src, size); From dovecot at dovecot.org Wed Oct 24 11:52:25 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 11:52:25 +0300 Subject: dovecot-2.2: raw storage: Don't set NAMESPACE_FLAG_INBOX_USER fo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5b0ab07024ca changeset: 15251:5b0ab07024ca user: Timo Sirainen date: Wed Oct 24 11:52:14 2012 +0300 description: raw storage: Don't set NAMESPACE_FLAG_INBOX_USER for the created namespace. diffstat: src/lib-storage/index/raw/raw-storage.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r b89dae6aead4 -r 5b0ab07024ca src/lib-storage/index/raw/raw-storage.c --- a/src/lib-storage/index/raw/raw-storage.c Wed Oct 24 11:30:09 2012 +0300 +++ b/src/lib-storage/index/raw/raw-storage.c Wed Oct 24 11:52:14 2012 +0300 @@ -33,6 +33,9 @@ ns_set->separator = "/"; ns = mail_namespaces_init_empty(user); + /* raw storage doesn't have INBOX. We especially don't want LIST to + return INBOX. */ + ns->flags &= ~NAMESPACE_FLAG_INBOX_USER; ns->flags |= NAMESPACE_FLAG_NOQUOTA | NAMESPACE_FLAG_NOACL; ns->set = ns_set; if (mail_storage_create(ns, "raw", 0, &error) < 0) From dovecot at dovecot.org Wed Oct 24 12:15:31 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 12:15:31 +0300 Subject: dovecot-2.2: mkdir_parents_chown(): If gid is set, make sure set... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7f95f182560 changeset: 15252:a7f95f182560 user: Timo Sirainen date: Wed Oct 24 12:15:19 2012 +0300 description: mkdir_parents_chown(): If gid is set, make sure setgid-bit isn't copied from parent. diffstat: src/lib/mkdir-parents.c | 12 ++++++++++++ src/lib/mkdir-parents.h | 4 +++- 2 files changed, 15 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 5b0ab07024ca -r a7f95f182560 src/lib/mkdir-parents.c --- a/src/lib/mkdir-parents.c Wed Oct 24 11:52:14 2012 +0300 +++ b/src/lib/mkdir-parents.c Wed Oct 24 12:15:19 2012 +0300 @@ -65,6 +65,18 @@ i_error("%s) failed: %m", str_c(str)); return -1; } + if (gid != (gid_t)-1 && (mode & S_ISGID) == 0) { + /* make sure the directory doesn't have setgid bit enabled + (in case its parent had) */ + if (chmod(path, mode) < 0) { + orig_errno = errno; + if (rmdir(path) < 0) + i_error("rmdir(%s) failed: %m", path); + errno = orig_errno; + i_error("chmod(%s) failed: %m", path); + return -1; + } + } return 0; } diff -r 5b0ab07024ca -r a7f95f182560 src/lib/mkdir-parents.h --- a/src/lib/mkdir-parents.h Wed Oct 24 11:52:14 2012 +0300 +++ b/src/lib/mkdir-parents.h Wed Oct 24 12:15:19 2012 +0300 @@ -8,7 +8,9 @@ /* Like mkdir_parents(), but use the given uid/gid for newly created directories. (uid_t)-1 or (gid_t)-1 can be used to indicate that it - doesn't need to be changed. */ + doesn't need to be changed. If gid isn't (gid_t)-1 and the parent directory + had setgid-bit enabled, it's removed unless explicitly included in the + mode. */ int mkdir_parents_chown(const char *path, mode_t mode, uid_t uid, gid_t gid); /* Like mkdir_parents_chown(), but change only group. If chown() fails with EACCES, use gid_origin in the error message. */ From dovecot at dovecot.org Wed Oct 24 12:23:46 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 12:23:46 +0300 Subject: dovecot-2.2: lib-storage: When creating user directory under dom... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fa8fa246189a changeset: 15253:fa8fa246189a user: Timo Sirainen date: Wed Oct 24 12:23:36 2012 +0300 description: lib-storage: When creating user directory under domain, set its setgid-bit on. The previous mkdir_parents_chown() change changed this behavior. I'm not entirely sure if the setgid-bit is a good idea or not in here, but we'll just continue the existing behavior. diffstat: src/lib-storage/mailbox-list.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a7f95f182560 -r fa8fa246189a src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Wed Oct 24 12:15:19 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Wed Oct 24 12:23:36 2012 +0300 @@ -1001,6 +1001,7 @@ if (perm->file_create_gid == (gid_t)-1 && (perm->dir_create_mode & S_ISGID) == 0) { /* change the group for user directories */ + perm->dir_create_mode |= S_ISGID; perm->file_create_gid = getegid(); perm->file_create_gid_origin = "egid"; perm->gid_origin_is_mailbox_path = FALSE; From dovecot at dovecot.org Wed Oct 24 12:47:47 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 12:47:47 +0300 Subject: dovecot-2.2: lib-storage: When mkdir()ing home, copy permissions... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/40b6534ddae3 changeset: 15254:40b6534ddae3 user: Timo Sirainen date: Wed Oct 24 12:47:34 2012 +0300 description: lib-storage: When mkdir()ing home, copy permissions from parent if it has setgid-bit and we're using %h/~. When using %d/%n instead of %h in the location directories, use the earlier methods. diffstat: src/lib-storage/mailbox-list.c | 88 ++++++++++++++++++++++++++++++----------- 1 files changed, 63 insertions(+), 25 deletions(-) diffs (163 lines): diff -r fa8fa246189a -r 40b6534ddae3 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Wed Oct 24 12:23:36 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Wed Oct 24 12:47:34 2012 +0300 @@ -207,14 +207,16 @@ return 0; } -static int fix_path(struct mail_user *user, const char *path, +static int fix_path(struct mail_user *user, const char *path, bool expand_home, const char **path_r, const char **error_r) { size_t len = strlen(path); if (len > 1 && path[len-1] == '/') path = t_strndup(path, len-1); - if (path[0] == '~' && path[1] != '/' && path[1] != '\0') { + if (!expand_home) { + /* no ~ expansion */ + } else if (path[0] == '~' && path[1] != '/' && path[1] != '\0') { /* ~otheruser/dir */ if (home_try_expand(&path) < 0) { *error_r = t_strconcat( @@ -253,9 +255,11 @@ return str; } -int mailbox_list_settings_parse(struct mail_user *user, const char *data, - struct mailbox_list_settings *set_r, - const char **error_r) +static int +mailbox_list_settings_parse_full(struct mail_user *user, const char *data, + bool expand_home, + struct mailbox_list_settings *set_r, + const char **error_r) { const char *const *tmp, *key, *value, **dest, *str, *error; @@ -271,7 +275,7 @@ /* */ tmp = t_strsplit(data, ":"); str = split_next_arg(&tmp); - if (fix_path(user, str, &set_r->root_dir, &error) < 0) { + if (fix_path(user, str, expand_home, &set_r->root_dir, &error) < 0) { *error_r = t_strconcat(error, "mail root dir in: ", data, NULL); return -1; } @@ -323,7 +327,7 @@ *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; } - if (fix_path(user, value, dest, &error) < 0) { + if (fix_path(user, value, expand_home, dest, &error) < 0) { *error_r = t_strconcat(error, key, " in: ", data, NULL); return -1; } @@ -334,6 +338,14 @@ return 0; } +int mailbox_list_settings_parse(struct mail_user *user, const char *data, + struct mailbox_list_settings *set_r, + const char **error_r) +{ + return mailbox_list_settings_parse_full(user, data, TRUE, + set_r, error_r); +} + const char *mailbox_list_get_unexpanded_path(struct mailbox_list *list, enum mailbox_list_path_type type) { @@ -367,7 +379,8 @@ if (p == NULL) return ""; - if (mailbox_list_settings_parse(user, p + 1, &set, &error) < 0) + if (mailbox_list_settings_parse_full(user, p + 1, FALSE, + &set, &error) < 0) return ""; if (mailbox_list_set_get_root_path(&set, type, &path) <= 0) return ""; @@ -963,33 +976,50 @@ { const char *expanded, *unexpanded, *root_dir, *p; struct stat st; + bool home = FALSE; /* get the directory path up to last %variable. for example unexpanded path may be "/var/mail/%d/%2n/%n/Maildir", and we want to get expanded="/var/mail/domain/nn" */ unexpanded = mailbox_list_get_unexpanded_path(list, type); p = strrchr(unexpanded, '%'); - if (p == NULL) - expanded = ""; - else { + if ((p == unexpanded && p[1] == 'h') || + (p == NULL && unexpanded[0] == '~')) { + /* home directory used */ + if (!mailbox_list_get_root_path(list, type, &expanded)) + i_unreached(); + home = TRUE; + } else if (p == NULL) { + return 0; + } else { while (p != unexpanded && *p != '/') p--; if (p == unexpanded) - expanded = ""; - else { - if (!mailbox_list_get_root_path(list, type, &expanded)) - i_unreached(); - expanded = get_expanded_path(unexpanded, p, expanded); - } + return 0; + + if (!mailbox_list_get_root_path(list, type, &expanded)) + i_unreached(); + expanded = get_expanded_path(unexpanded, p, expanded); + if (*expanded == '\0') + return 0; } - if (*expanded != '\0') { - /* up to this directory get the permissions from the first - parent directory that exists, if it has setgid bit - enabled. */ - if (mailbox_list_stat_parent(expanded, &root_dir, &st, - error_r) < 0) - return -1; - if ((st.st_mode & S_ISGID) != 0 && root_dir != expanded) { + /* get the first existing parent directory's permissions */ + if (mailbox_list_stat_parent(expanded, &root_dir, &st, error_r) < 0) + return -1; + + /* if the parent directory doesn't have setgid-bit enabled, we don't + copy any permissions from it. */ + if ((st.st_mode & S_ISGID) == 0) + return 0; + + if (!home) { + /* assuming we have e.g. /var/vmail/%d/%n directory, here we + want to create up to /var/vmail/%d with permissions from + the parent directory. we never want to create the %n + directory itself. */ + if (root_dir == expanded) { + /* this is the %n directory */ + } else { if (mkdir_parents_chgrp(expanded, st.st_mode, (gid_t)-1, root_dir) < 0 && errno != EEXIST) { @@ -1006,6 +1036,14 @@ perm->file_create_gid_origin = "egid"; perm->gid_origin_is_mailbox_path = FALSE; } + } else { + /* when using %h and the parent has setgid-bit, + copy the permissions from it for the home we're creating */ + perm->file_create_mode = st.st_mode & 0666; + perm->dir_create_mode = st.st_mode; + perm->file_create_gid = (gid_t)-1; + perm->file_create_gid_origin = "parent"; + perm->gid_origin_is_mailbox_path = FALSE; } return 0; } From dovecot at dovecot.org Wed Oct 24 13:04:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Oct 2012 13:04:49 +0300 Subject: dovecot-2.2: NEWS updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e34a51082c11 changeset: 15255:e34a51082c11 user: Timo Sirainen date: Wed Oct 24 13:04:38 2012 +0300 description: NEWS updated diffstat: NEWS | 13 ++++++++++--- 1 files changed, 10 insertions(+), 3 deletions(-) diffs (34 lines): diff -r 40b6534ddae3 -r e34a51082c11 NEWS --- a/NEWS Wed Oct 24 12:47:34 2012 +0300 +++ b/NEWS Wed Oct 24 13:04:38 2012 +0300 @@ -1,13 +1,15 @@ v2.2.UNSTABLE 2012-xx-xx Timo Sirainen + * When creating home directories, the permissions are copied from the + parent directory if it has setgid-bit set. For full details, see + http://wiki2.dovecot.org/SharedMailboxes/Permissions * "doveadm auth" command was renamed to "doveadm auth test" + Implemented IMAP MOVE and BINARY extensions - + Implemented IMAP CATENATE extension (by Stephan Bosch) + + Implemented IMAP CATENATE, URLAUTH and URLAUTH=BINARY extensions + (by Stephan Bosch). + Implemented IMAP NOTIFY extension. Requires mailbox_list_index=yes to be enabled. - + Improved mailbox list indexes. They should be usable now, although - still disabled by default. + Redesigned and rewritten dsync. The new design makes the syncing faster, more reliable and more featureful. The new dsync protocol isn't backwards compatible with old dsync versions (but is designed @@ -16,6 +18,11 @@ mailboxes by using a private index. It can be enabled by adding :INDEXPVT= to mail location. This should be used instead of :INDEX also for Maildir/mbox to improve performance. + + Improved mailbox list indexes. They should be usable now, although + still disabled by default. + + Added LAYOUT=index. The mailbox directories are created using their + GUIDs in the filesystem, while the actual GUID <-> name mapping + exists only in the index. + LMTP proxy: Implemented XCLIENT extension for passing remote IP address through proxy. From dovecot at dovecot.org Fri Oct 26 10:36:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 10:36:55 +0300 Subject: dovecot-2.2: dsync: Crashfix for scanning keyword changes in tra... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce3f3006383b changeset: 15256:ce3f3006383b user: Timo Sirainen date: Wed Oct 24 15:05:15 2012 +0300 description: dsync: Crashfix for scanning keyword changes in transaction log. diffstat: src/doveadm/dsync/dsync-transaction-log-scan.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e34a51082c11 -r ce3f3006383b src/doveadm/dsync/dsync-transaction-log-scan.c --- a/src/doveadm/dsync/dsync-transaction-log-scan.c Wed Oct 24 13:04:38 2012 +0300 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Wed Oct 24 15:05:15 2012 +0300 @@ -252,7 +252,7 @@ uids = CONST_PTR_OFFSET(rec, uids_offset); end = CONST_PTR_OFFSET(rec, hdr->size); - for (; uids <= end; uids += 2) { + for (; uids < end; uids += 2) { for (uid = uids[0]; uid <= uids[1]; uid++) { if (!export_change_get(ctx, uid, DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, From dovecot at dovecot.org Fri Oct 26 10:36:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 10:36:55 +0300 Subject: dovecot-2.2: lib-index: mail_index_transaction_lookup_latest_key... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67af7122a522 changeset: 15258:67af7122a522 user: Timo Sirainen date: Fri Oct 26 10:36:43 2012 +0300 description: lib-index: mail_index_transaction_lookup_latest_keywords() didn't work properly. It wasn't actually looking up the latest keywords. diffstat: src/lib-index/mail-index-transaction.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (16 lines): diff -r 60f0cb48fdb2 -r 67af7122a522 src/lib-index/mail-index-transaction.c --- a/src/lib-index/mail-index-transaction.c Wed Oct 24 15:05:40 2012 +0300 +++ b/src/lib-index/mail-index-transaction.c Fri Oct 26 10:36:43 2012 +0300 @@ -101,9 +101,9 @@ mail_index_refresh(t->view->index); t->latest_view = mail_index_view_open(t->view->index); } - mail_index_lookup_uid(t->view, seq, &uid); - if (mail_index_lookup_seq(t->view, uid, &latest_seq)) - mail_index_lookup_keywords(t->view, latest_seq, keywords); + mail_index_lookup_uid(t->latest_view, seq, &uid); + if (mail_index_lookup_seq(t->latest_view, uid, &latest_seq)) + mail_index_lookup_keywords(t->latest_view, latest_seq, keywords); } static int From dovecot at dovecot.org Fri Oct 26 10:36:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 10:36:55 +0300 Subject: dovecot-2.2: doveadm backup: Revert all local changes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/60f0cb48fdb2 changeset: 15257:60f0cb48fdb2 user: Timo Sirainen date: Wed Oct 24 15:05:40 2012 +0300 description: doveadm backup: Revert all local changes. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 2 + src/doveadm/dsync/dsync-mailbox-import.c | 47 ++++++++++++++++++++++++++++++++ src/doveadm/dsync/dsync-mailbox-import.h | 3 +- 3 files changed, 51 insertions(+), 1 deletions(-) diffs (103 lines): diff -r ce3f3006383b -r 60f0cb48fdb2 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Wed Oct 24 15:05:15 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Wed Oct 24 15:05:40 2012 +0300 @@ -195,6 +195,8 @@ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN; if (brain->mails_have_guids) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; + if (brain->backup_recv) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES; brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->log_scan, diff -r ce3f3006383b -r 60f0cb48fdb2 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Wed Oct 24 15:05:15 2012 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Wed Oct 24 15:05:40 2012 +0300 @@ -81,6 +81,7 @@ unsigned int want_mail_requests:1; unsigned int mails_have_guids:1; unsigned int master_brain:1; + unsigned int revert_local_changes:1; }; static void @@ -158,6 +159,8 @@ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0; importer->master_brain = (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0; + importer->revert_local_changes = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0; mailbox_get_open_status(importer->box, STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ, @@ -635,6 +638,44 @@ } static void +dsync_mailbox_import_replace_flags(struct mail *mail, + const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + struct mail_keywords *kw; + const char *const *changes, *name; + unsigned int i, count; + + if (array_is_created(&change->keyword_changes)) + changes = array_get(&change->keyword_changes, &count); + else { + changes = NULL; + count = 0; + } + t_array_init(&keywords, count+1); + for (i = 0; i < count; i++) { + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_FINAL: + name = changes[i]+1; + array_append(&keywords, &name, 1); + break; + case KEYWORD_CHANGE_REMOVE: + break; + } + } + array_append_zero(&keywords); + + kw = mailbox_keywords_create_valid(mail->box, array_idx(&keywords, 0)); + mail_update_keywords(mail, MODIFY_REPLACE, kw); + mailbox_keywords_unref(&kw); + + mail_update_flags(mail, MODIFY_REPLACE, + change->add_flags | change->final_flags); + mail_update_modseq(mail, change->modseq); +} + +static void dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer, const struct dsync_mail_change *change) { @@ -656,6 +697,12 @@ mail = importer->mail; } + if (importer->revert_local_changes) { + /* dsync backup: just make the local look like remote. */ + dsync_mailbox_import_replace_flags(mail, change); + return; + } + local_change = hash_table_lookup(importer->local_changes, POINTER_CAST(change->uid)); if (local_change == NULL) { diff -r ce3f3006383b -r 60f0cb48fdb2 src/doveadm/dsync/dsync-mailbox-import.h --- a/src/doveadm/dsync/dsync-mailbox-import.h Wed Oct 24 15:05:15 2012 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Wed Oct 24 15:05:40 2012 +0300 @@ -4,7 +4,8 @@ enum dsync_mailbox_import_flags { DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01, DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, - DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x04 + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x04, + DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x08 }; struct mailbox; From dovecot at dovecot.org Fri Oct 26 11:07:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:07:49 +0300 Subject: dovecot-2.2: doveadm backup: Don't assert-crash trying to delete... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83cdfd0bc79d changeset: 15259:83cdfd0bc79d user: Timo Sirainen date: Fri Oct 26 11:07:43 2012 +0300 description: doveadm backup: Don't assert-crash trying to delete noselect-mailboxes too early. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 67af7122a522 -r 83cdfd0bc79d src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Oct 26 10:36:43 2012 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Fri Oct 26 11:07:43 2012 +0300 @@ -932,7 +932,8 @@ ret = strcmp(node->name, wanted_node->name); if (ret < 0) { /* node shouldn't exist */ - if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) sync_delete_mailbox_node(ctx, tree, node); node = node->next; } else if (ret > 0) { @@ -940,7 +941,8 @@ wanted_node = wanted_node->next; } else { if (sync_is_wrong_mailbox(node, wanted_node) && - node->existence == DSYNC_MAILBOX_NODE_EXISTS) + node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) sync_delete_mailbox_node(ctx, tree, node); node = node->next; wanted_node = wanted_node->next; @@ -953,7 +955,8 @@ tree, wanted_tree, node->first_child, NULL); } - if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) sync_delete_mailbox_node(ctx, tree, node); } } From dovecot at dovecot.org Fri Oct 26 11:12:41 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:12:41 +0300 Subject: dovecot-2.2: dbox: Log a better error if we have external attach... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26236c4e5736 changeset: 15260:26236c4e5736 user: Timo Sirainen date: Fri Oct 26 11:12:36 2012 +0300 description: dbox: Log a better error if we have external attachments, but mail_attachment_dir is unset. diffstat: src/lib-storage/index/dbox-common/dbox-attachment.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (17 lines): diff -r 83cdfd0bc79d -r 26236c4e5736 src/lib-storage/index/dbox-common/dbox-attachment.c --- a/src/lib-storage/index/dbox-common/dbox-attachment.c Fri Oct 26 11:07:43 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.c Fri Oct 26 11:12:36 2012 +0300 @@ -150,6 +150,13 @@ *error_r = NULL; + if (*file->storage->attachment_dir == '\0') { + mail_storage_set_critical(&file->storage->storage, + "%s contains references to external attachments, " + "but mail_attachment_dir is unset", file->cur_path); + return -1; + } + t_array_init(&extrefs_arr, 16); if (!dbox_attachment_parse_extref_real(ext_refs, pool_datastack_create(), &extrefs_arr)) { From dovecot at dovecot.org Fri Oct 26 11:18:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:18:02 +0300 Subject: dovecot-2.1: lib-storage: Listing multiple mailbox patterns may ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/8f351303887c changeset: 14783:8f351303887c user: Timo Sirainen date: Fri Oct 26 11:17:51 2012 +0300 description: lib-storage: Listing multiple mailbox patterns may have returned duplicates with fs layout. diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 49bb6cc43d03 -r 8f351303887c src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Mon Oct 22 18:59:20 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:17:51 2012 +0300 @@ -447,7 +447,7 @@ /* sort the root dirs so that /foo is before /foo/bar */ array_sort(&ctx->roots, i_strcmp_p); /* remove /foo/bar when there already exists /foo parent */ - for (i = 1; i < array_count(&ctx->roots); i++) { + for (i = 1; i < array_count(&ctx->roots); ) { parentp = array_idx(&ctx->roots, i-1); childp = array_idx(&ctx->roots, i); parentlen = strlen(*parentp); @@ -456,6 +456,8 @@ (*childp)[parentlen] == ctx->sep || (*childp)[parentlen] == '\0')) array_delete(&ctx->roots, i, 1); + else + i++; } } From dovecot at dovecot.org Fri Oct 26 11:37:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:37:23 +0300 Subject: dovecot-2.1: Increased initial memory pool size Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b41d2eec320b changeset: 14784:b41d2eec320b user: Timo Sirainen date: Fri Oct 26 11:33:22 2012 +0300 description: Increased initial memory pool size diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8f351303887c -r b41d2eec320b src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:17:51 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:33:22 2012 +0300 @@ -490,7 +490,7 @@ flags); } - pool = pool_alloconly_create("mailbox list fs iter", 1024); + pool = pool_alloconly_create("mailbox list fs iter", 2048); ctx = p_new(pool, struct fs_list_iterate_context, 1); ctx->ctx.pool = pool; ctx->ctx.list = _list; From dovecot at dovecot.org Fri Oct 26 11:37:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:37:23 +0300 Subject: dovecot-2.1: lib-index: If cache file unexpectedly shrinks in he... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7fde3830215e changeset: 14785:7fde3830215e user: Timo Sirainen date: Fri Oct 26 11:34:25 2012 +0300 description: lib-index: If cache file unexpectedly shrinks in header lookup, log an error. diffstat: src/lib-index/mail-cache-lookup.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (22 lines): diff -r b41d2eec320b -r 7fde3830215e src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Fri Oct 26 11:33:22 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Fri Oct 26 11:34:25 2012 +0300 @@ -538,9 +538,16 @@ /* then start filling dest buffer from the headers */ for (i = 0; i < count; i++) { - if (mail_cache_map(cache, lines[i].data->offset, - lines[i].data->data_size, &data) <= 0) + ret = mail_cache_map(cache, lines[i].data->offset, + lines[i].data->data_size, &data); + if (ret <= 0) { + if (ret < 0) + return -1; + + mail_cache_set_corrupted(cache, + "header record unexpectedly points outside file"); return -1; + } start = data; end = start + lines[i].data->data_size; From dovecot at dovecot.org Fri Oct 26 11:37:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:37:23 +0300 Subject: dovecot-2.1: lib-index: After recent cache changes, cache was of... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/3700fb6f8a42 changeset: 14786:3700fb6f8a42 user: Timo Sirainen date: Fri Oct 26 11:37:07 2012 +0300 description: lib-index: After recent cache changes, cache was often wrongly being thought of as unusable diffstat: src/lib-index/mail-cache.c | 28 +++++++++++++++++++--------- 1 files changed, 19 insertions(+), 9 deletions(-) diffs (88 lines): diff -r 7fde3830215e -r 3700fb6f8a42 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Fri Oct 26 11:34:25 2012 +0300 +++ b/src/lib-index/mail-cache.c Fri Oct 26 11:37:07 2012 +0300 @@ -267,11 +267,13 @@ static int mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, - const void *data, bool copy_hdr) + const void *hdr_data, bool copy_hdr) { + const struct mail_cache_header *hdr = hdr_data; + if (offset == 0) { - const struct mail_cache_header *hdr = data; - + /* verify the header validity only with offset=0. this way + we won't waste time re-verifying it all the time */ if (!mail_cache_verify_header(cache, hdr)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && @@ -279,14 +281,18 @@ cache->hdr->file_seq : 0; return -1; } + } + if (hdr_data != NULL) { if (!copy_hdr) - cache->hdr = data; + cache->hdr = hdr; else { - memcpy(&cache->hdr_ro_copy, data, + memcpy(&cache->hdr_ro_copy, hdr, sizeof(cache->hdr_ro_copy)); cache->hdr = &cache->hdr_ro_copy; } mail_cache_update_need_compress(cache); + } else { + i_assert(cache->hdr != NULL); } if (offset + size > cache->mmap_length) @@ -298,6 +304,7 @@ mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { + const void *hdr_data; void *data; ssize_t ret; @@ -309,7 +316,8 @@ /* already mapped */ *data_r = CONST_PTR_OFFSET(cache->read_buf->data, offset - cache->read_offset); - return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); } else { buffer_set_used_size(cache->read_buf, 0); } @@ -330,7 +338,8 @@ cache->mmap_length = offset + size; *data_r = data; - return mail_cache_map_finish(cache, offset, size, data, TRUE); + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); } int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, @@ -367,7 +376,7 @@ &cache->mmap_length); *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(data, offset); - return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + return mail_cache_map_finish(cache, offset, size, data, TRUE); } if (offset < cache->mmap_length && @@ -402,7 +411,8 @@ } *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(cache->mmap_base, offset); - return mail_cache_map_finish(cache, offset, size, *data_r, FALSE); + return mail_cache_map_finish(cache, offset, size, + cache->mmap_base, FALSE); } static int mail_cache_try_open(struct mail_cache *cache) From dovecot at dovecot.org Fri Oct 26 11:43:15 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 11:43:15 +0300 Subject: dovecot-2.1: lib-index: Optimize cache file reads with MAIL_INDE... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/0efb98659a1f changeset: 14787:0efb98659a1f user: Timo Sirainen date: Fri Oct 26 11:43:05 2012 +0300 description: lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY diffstat: src/lib-index/mail-cache-fields.c | 2 +- src/lib-index/mail-cache.c | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletions(-) diffs (38 lines): diff -r 3700fb6f8a42 -r 0efb98659a1f src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Fri Oct 26 11:37:07 2012 +0300 +++ b/src/lib-index/mail-cache-fields.c Fri Oct 26 11:43:05 2012 +0300 @@ -232,7 +232,7 @@ offset = next_offset; invalidate = TRUE; - if (cache->mmap_base != NULL) { + if (cache->mmap_base != NULL || cache->map_with_read) { ret = mail_cache_map(cache, offset, sizeof(*field_hdr), &data); if (ret <= 0) { diff -r 3700fb6f8a42 -r 0efb98659a1f src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Fri Oct 26 11:37:07 2012 +0300 +++ b/src/lib-index/mail-cache.c Fri Oct 26 11:43:05 2012 +0300 @@ -13,6 +13,8 @@ #include +#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096 + void mail_cache_set_syscall_error(struct mail_cache *cache, const char *function) { @@ -321,6 +323,13 @@ } else { buffer_set_used_size(cache->read_buf, 0); } + if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) { + /* we can usually read the fields header after the cache + header. we need them both, so try to read them all with one + pread() call. */ + size = MAIL_CACHE_MIN_HEADER_READ_SIZE; + } + data = buffer_append_space_unsafe(cache->read_buf, size); ret = pread(cache->fd, data, size, offset); if (ret < 0) { From dovecot at dovecot.org Fri Oct 26 12:09:32 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 12:09:32 +0300 Subject: dovecot-2.1: stats: Refresh user statistics during long maildir ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/b1b693a69c5f changeset: 14788:b1b693a69c5f user: Timo Sirainen date: Fri Oct 26 12:09:03 2012 +0300 description: stats: Refresh user statistics during long maildir syncs. diffstat: src/plugins/stats/stats-plugin.c | 36 ++++++++++++++++++++++++++++++++++++ 1 files changed, 36 insertions(+), 0 deletions(-) diffs (60 lines): diff -r 0efb98659a1f -r b1b693a69c5f src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Fri Oct 26 11:43:05 2012 +0300 +++ b/src/plugins/stats/stats-plugin.c Fri Oct 26 12:09:03 2012 +0300 @@ -33,6 +33,13 @@ struct mailbox_transaction_stats prev_stats; }; +struct stats_storage { + union mail_storage_module_context module_ctx; + + struct mail_storage_callbacks old_callbacks; + void *old_context; +}; + struct stats_mailbox { union mailbox_module_context module_ctx; }; @@ -431,6 +438,33 @@ return ret; } +static void +stats_notify_ok(struct mailbox *box, const char *text, void *context) +{ + struct stats_storage *sstorage = STATS_CONTEXT(box->storage); + + /* most importantly we want to refresh stats for very long running + mailbox syncs */ + session_stats_refresh(box->storage->user); + + if (sstorage->old_callbacks.notify_ok != NULL) + sstorage->old_callbacks.notify_ok(box, text, context); +} + +static void stats_register_notify_callbacks(struct mail_storage *storage) +{ + struct stats_storage *sstorage = STATS_CONTEXT(storage); + + if (sstorage != NULL) + return; + + sstorage = p_new(storage->pool, struct stats_storage, 1); + sstorage->old_callbacks = storage->callbacks; + storage->callbacks.notify_ok = stats_notify_ok; + + MODULE_CONTEXT_SET(storage, stats_storage_module, sstorage); +} + static void stats_mailbox_allocated(struct mailbox *box) { struct mailbox_vfuncs *v = box->vlast; @@ -440,6 +474,8 @@ if (suser == NULL) return; + stats_register_notify_callbacks(box->storage); + sbox = p_new(box->pool, struct stats_mailbox, 1); sbox->module_ctx.super = *v; box->vlast = &sbox->module_ctx.super; From dovecot at dovecot.org Fri Oct 26 13:06:13 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 26 Oct 2012 13:06:13 +0300 Subject: dovecot-2.1: lib-storage: Fixed listing layout=fs when namespace... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/22875bcaa952 changeset: 14789:22875bcaa952 user: Timo Sirainen date: Fri Oct 26 13:05:43 2012 +0300 description: lib-storage: Fixed listing layout=fs when namespace prefix part included wildcards. diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diffs (34 lines): diff -r b1b693a69c5f -r 22875bcaa952 src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 12:09:03 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 13:05:43 2012 +0300 @@ -395,13 +395,24 @@ for (patterns = ctx->valid_patterns; *patterns != NULL; patterns++) { pattern = *patterns; - for (p = last = pattern; *p != '\0'; p++) { - if (*p == '%' || *p == '*') - break; - if (*p == ns_sep) - last = p; + if (strncmp(pattern, ns->prefix, ns->prefix_len) != 0) { + /* typically e.g. prefix=foo/bar/, pattern=foo/%/% + we'll use root="" for this. + + it might of course also be pattern=foo/%/prefix/% + where we could optimize with root=prefix, but + probably too much trouble to implement. */ + prefix_vname = ""; + p = last = pattern; + } else { + for (p = last = pattern; *p != '\0'; p++) { + if (*p == '%' || *p == '*') + break; + if (*p == ns_sep) + last = p; + } + prefix_vname = t_strdup_until(pattern, last); } - prefix_vname = t_strdup_until(pattern, last); if (p == last+1 && *pattern == ns_sep) root = "/"; From dovecot at dovecot.org Mon Oct 29 12:40:27 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 12:40:27 +0200 Subject: dovecot-2.2: acl: Avoid assert-crashing when trying to access sh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d0d3cc20bdb changeset: 15261:4d0d3cc20bdb user: Timo Sirainen date: Mon Oct 29 12:40:14 2012 +0200 description: acl: Avoid assert-crashing when trying to access shared namespace root "mailboxes". diffstat: src/plugins/acl/acl-mailbox.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (17 lines): diff -r 26236c4e5736 -r 4d0d3cc20bdb src/plugins/acl/acl-mailbox.c --- a/src/plugins/acl/acl-mailbox.c Fri Oct 26 11:12:36 2012 +0300 +++ b/src/plugins/acl/acl-mailbox.c Mon Oct 29 12:40:14 2012 +0200 @@ -524,6 +524,13 @@ return; } + if (box->list->ns->type == MAIL_NAMESPACE_TYPE_SHARED && + (box->list->ns->flags & NAMESPACE_FLAG_AUTOCREATED) == 0) { + /* this is the root shared namespace, which itself doesn't + have any existing mailboxes. */ + return; + } + abox = p_new(box->pool, struct acl_mailbox, 1); abox->module_ctx.super = *v; box->vlast = &abox->module_ctx.super; From dovecot at dovecot.org Mon Oct 29 12:45:01 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 12:45:01 +0200 Subject: dovecot-2.2: lib-index: Avoid assert-crashing when syncing an ol... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0a6fe3aa61f changeset: 15262:b0a6fe3aa61f user: Timo Sirainen date: Mon Oct 29 12:43:29 2012 +0200 description: lib-index: Avoid assert-crashing when syncing an old "keyword reset" from transaction log. diffstat: src/lib-index/mail-index-transaction-update.c | 14 +++++++++++++- 1 files changed, 13 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 4d0d3cc20bdb -r b0a6fe3aa61f src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Mon Oct 29 12:40:14 2012 +0200 +++ b/src/lib-index/mail-index-transaction-update.c Mon Oct 29 12:43:29 2012 +0200 @@ -1014,9 +1014,21 @@ keyword_update_remove_existing(struct mail_index_transaction *t, uint32_t seq) { ARRAY_TYPE(keyword_indexes) keywords; + uint32_t i, keywords_count; t_array_init(&keywords, 32); - mail_index_transaction_lookup_latest_keywords(t, seq, &keywords); + if (t->view->v.lookup_full == NULL) { + /* syncing is saving a list of changes into this transaction. + the seq is actual an uid, so we can't lookup the existing + keywords. we shouldn't get here unless we're reading + pre-v2.2 keyword-reset records from .log files. so we don't + really care about performance that much here, */ + keywords_count = array_count(&t->view->index->keywords); + for (i = 0; i < keywords_count; i++) + array_append(&keywords, &i, 1); + } else { + mail_index_transaction_lookup_latest_keywords(t, seq, &keywords); + } if (array_count(&keywords) == 0) return NULL; return mail_index_keywords_create_from_indexes(t->view->index, From dovecot at dovecot.org Mon Oct 29 12:45:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 12:45:02 +0200 Subject: dovecot-2.2: lib-index: MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_U... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1a3348e3892f changeset: 15263:1a3348e3892f user: Timo Sirainen date: Mon Oct 29 12:44:43 2012 +0200 description: lib-index: MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES didn't work well enough for keywords. Removed the code that attempted to optimize some CPU usage at the cost of writing too many changes. diffstat: src/lib-index/mail-index-transaction-update.c | 11 ----------- 1 files changed, 0 insertions(+), 11 deletions(-) diffs (28 lines): diff -r b0a6fe3aa61f -r 1a3348e3892f src/lib-index/mail-index-transaction-update.c --- a/src/lib-index/mail-index-transaction-update.c Mon Oct 29 12:43:29 2012 +0200 +++ b/src/lib-index/mail-index-transaction-update.c Mon Oct 29 12:44:43 2012 +0200 @@ -963,7 +963,6 @@ enum modify_type modify_type, struct mail_keywords *keywords) { - struct mail_index_transaction_keyword_update *u; ARRAY_TYPE(keyword_indexes) existing; const unsigned int *existing_idx; unsigned int i, j, existing_count; @@ -978,16 +977,6 @@ return TRUE; for (i = 0; i < keywords->count; i++) { - u = array_idx_modifiable(&t->keyword_updates, - keywords->idx[i]); - if (array_is_created(&u->add_seq) || - array_is_created(&u->remove_seq)) { - /* we've already modified this keyword in the - transaction. don't bother checking it further, - because we can't avoid the changes anyway. */ - return TRUE; - } - found = FALSE; for (j = 0; j < existing_count; j++) { if (existing_idx[j] == keywords->idx[i]) { From dovecot at dovecot.org Mon Oct 29 13:01:51 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 13:01:51 +0200 Subject: dovecot-2.2: dsync: Make sure we're not trying to use uidvalidit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3d7edc15fcec changeset: 15264:3d7edc15fcec user: Timo Sirainen date: Mon Oct 29 13:00:36 2012 +0200 description: dsync: Make sure we're not trying to use uidvalidity=0 for existing mailboxes. diffstat: src/doveadm/dsync/dsync-mailbox-tree-fill.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 1a3348e3892f -r 3d7edc15fcec src/doveadm/dsync/dsync-mailbox-tree-fill.c --- a/src/doveadm/dsync/dsync-mailbox-tree-fill.c Mon Oct 29 12:44:43 2012 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c Mon Oct 29 13:00:36 2012 +0200 @@ -69,6 +69,7 @@ return -1; } } else { + i_assert(status.uidvalidity != 0); memcpy(node->mailbox_guid, metadata.guid, sizeof(node->mailbox_guid)); node->uid_validity = status.uidvalidity; From dovecot at dovecot.org Mon Oct 29 13:01:51 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 13:01:51 +0200 Subject: dovecot-2.2: lib-storage: Make sure mailbox was synced at least ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/41018c6fcc53 changeset: 15265:41018c6fcc53 user: Timo Sirainen date: Mon Oct 29 13:01:37 2012 +0200 description: lib-storage: Make sure mailbox was synced at least once when getting status/metadata. diffstat: src/lib-storage/index/index-status.c | 6 ++++++ src/lib-storage/mail-storage.h | 3 +++ 2 files changed, 9 insertions(+), 0 deletions(-) diffs (36 lines): diff -r 3d7edc15fcec -r 41018c6fcc53 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Mon Oct 29 13:00:36 2012 +0200 +++ b/src/lib-storage/index/index-status.c Mon Oct 29 13:01:37 2012 +0200 @@ -36,6 +36,8 @@ if (!box->opened) { if (mailbox_open(box) < 0) return -1; + } + if (!box->synced) { if (mailbox_sync(box, 0) < 0) return -1; } @@ -357,6 +359,10 @@ if (mailbox_open(box) < 0) return -1; } + if (!box->synced && (items & MAILBOX_METADATA_SYNC_ITEMS) != 0) { + if (mailbox_sync(box, 0) < 0) + return -1; + } if ((items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) { if (get_metadata_virtual_size(box, metadata_r) < 0) diff -r 3d7edc15fcec -r 41018c6fcc53 src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Mon Oct 29 13:00:36 2012 +0200 +++ b/src/lib-storage/mail-storage.h Mon Oct 29 13:01:37 2012 +0200 @@ -85,6 +85,9 @@ MAILBOX_METADATA_CACHE_FIELDS = 0x04, MAILBOX_METADATA_PRECACHE_FIELDS = 0x08, MAILBOX_METADATA_BACKEND_NAMESPACE = 0x10 + /* metadata items that require mailbox to be synced at least once. */ +#define MAILBOX_METADATA_SYNC_ITEMS \ + (MAILBOX_METADATA_GUID | MAILBOX_METADATA_VIRTUAL_SIZE) }; enum mailbox_search_result_flags { From dovecot at dovecot.org Mon Oct 29 13:23:41 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 13:23:41 +0200 Subject: dovecot-2.2: lib-storage: Check and log stream errors when parsi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99965e11d9e1 changeset: 15266:99965e11d9e1 user: Timo Sirainen date: Mon Oct 29 13:23:30 2012 +0200 description: lib-storage: Check and log stream errors when parsing/searching messages. diffstat: src/lib-storage/index/index-mail-headers.c | 6 +++++- src/lib-storage/index/index-mail.c | 2 +- src/lib-storage/index/index-mail.h | 1 + src/lib-storage/index/index-search.c | 9 +++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diffs (80 lines): diff -r 41018c6fcc53 -r 99965e11d9e1 src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Mon Oct 29 13:01:37 2012 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Mon Oct 29 13:23:30 2012 +0200 @@ -430,6 +430,8 @@ hdr_parser_flags, index_mail_parse_header_cb, mail); } + if (index_mail_stream_check_failure(mail) < 0) + return -1; data->hdr_size_set = TRUE; data->access_part &= ~PARSE_HDR; @@ -475,15 +477,17 @@ mailbox_header_lookup_unref(&header_ctx); return -1; } + mailbox_header_lookup_unref(&header_ctx); if (mail->data.envelope == NULL && stream != NULL) { /* we got the headers from cache - parse them to get the envelope */ message_parse_header(stream, NULL, hdr_parser_flags, imap_envelope_parse_callback, mail); + if (index_mail_stream_check_failure(mail) < 0) + return -1; mail->data.save_envelope = FALSE; } - mailbox_header_lookup_unref(&header_ctx); if (mail->data.stream != NULL) i_stream_seek(mail->data.stream, old_offset); diff -r 41018c6fcc53 -r 99965e11d9e1 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Mon Oct 29 13:01:37 2012 +0200 +++ b/src/lib-storage/index/index-mail.c Mon Oct 29 13:23:30 2012 +0200 @@ -811,7 +811,7 @@ return 0; } -static int index_mail_stream_check_failure(struct index_mail *mail) +int index_mail_stream_check_failure(struct index_mail *mail) { if (mail->data.stream->stream_errno == 0) return 0; diff -r 41018c6fcc53 -r 99965e11d9e1 src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Mon Oct 29 13:01:37 2012 +0200 +++ b/src/lib-storage/index/index-mail.h Mon Oct 29 13:23:30 2012 +0200 @@ -223,6 +223,7 @@ void index_mail_set_cache_corrupted(struct mail *mail, enum mail_fetch_field field); int index_mail_opened(struct mail *mail, struct istream **stream); +int index_mail_stream_check_failure(struct index_mail *mail); struct index_mail *index_mail_get_index_mail(struct mail *mail); bool index_mail_get_cached_uoff_t(struct index_mail *mail, diff -r 41018c6fcc53 -r 99965e11d9e1 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Mon Oct 29 13:01:37 2012 +0200 +++ b/src/lib-storage/index/index-search.c Mon Oct 29 13:23:30 2012 +0200 @@ -626,6 +626,10 @@ ret = message_search_msg(msg_search_ctx, ctx->input, NULL); i_assert(ret >= 0 || ctx->input->stream_errno != 0); } + if (ctx->input->stream_errno != 0) { + mail_storage_set_critical(ctx->index_ctx->box->storage, + "read(%s) failed: %m", i_stream_get_name(ctx->input)); + } ARG_SET_RESULT(arg, ret); } @@ -686,6 +690,11 @@ } message_parse_header(input, NULL, hdr_parser_flags, search_header, &hdr_ctx); + if (input->stream_errno != 0) { + mail_storage_set_critical(ctx->box->storage, + "read(%s) failed: %m", i_stream_get_name(input)); + failed = TRUE; + } } } if (headers_ctx != NULL) From dovecot at dovecot.org Mon Oct 29 14:17:51 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:17:51 +0200 Subject: dovecot-2.2: lib-storage: Avoid assert-crashing when updating ma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b5a5f5139148 changeset: 15267:b5a5f5139148 user: Timo Sirainen date: Mon Oct 29 14:17:40 2012 +0200 description: lib-storage: Avoid assert-crashing when updating mailbox list index. diffstat: src/lib-storage/list/mailbox-list-index-status.c | 5 +++++ src/lib-storage/list/mailbox-list-index.h | 1 + 2 files changed, 6 insertions(+), 0 deletions(-) diffs (34 lines): diff -r 99965e11d9e1 -r b5a5f5139148 src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Mon Oct 29 13:23:30 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-status.c Mon Oct 29 14:17:40 2012 +0200 @@ -297,6 +297,9 @@ struct mailbox_status status; uint32_t seq, seq1, seq2; + if (ilist->syncing || ilist->updating_status) + return; + (void)mailbox_list_index_refresh(box->list); node = mailbox_list_index_lookup(box->list, box->name); @@ -331,7 +334,9 @@ status.highest_modseq = 1; } + ilist->updating_status = TRUE; (void)index_list_update(box, list_view, seq, &status); + ilist->updating_status = FALSE; } mail_index_view_close(&list_view); } diff -r 99965e11d9e1 -r b5a5f5139148 src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Mon Oct 29 13:23:30 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index.h Mon Oct 29 14:17:40 2012 +0200 @@ -108,6 +108,7 @@ unsigned int opened:1; unsigned int syncing:1; + unsigned int updating_status:1; unsigned int has_backing_store:1; }; From dovecot at dovecot.org Mon Oct 29 14:18:13 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:18:13 +0200 Subject: dovecot-2.2: imap: Commit temporary mail's transaction before fi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e8d7cfec5dc changeset: 15268:4e8d7cfec5dc user: Timo Sirainen date: Mon Oct 29 14:16:02 2012 +0200 description: imap: Commit temporary mail's transaction before finishing sync. Fixes assert-crash with mailbox list indexes. diffstat: src/imap/imap-sync.c | 10 ++++------ 1 files changed, 4 insertions(+), 6 deletions(-) diffs (32 lines): diff -r b5a5f5139148 -r 4e8d7cfec5dc src/imap/imap-sync.c --- a/src/imap/imap-sync.c Mon Oct 29 14:17:40 2012 +0200 +++ b/src/imap/imap-sync.c Mon Oct 29 14:16:02 2012 +0200 @@ -306,12 +306,15 @@ ctx->finished = TRUE; mail_free(&ctx->mail); + /* the transaction is used only for fetching modseqs/flags. + it can't really fail.. */ + (void)mailbox_transaction_commit(&ctx->t); + if (array_is_created(&ctx->expunges)) array_free(&ctx->expunges); if (mailbox_sync_deinit(&ctx->sync_ctx, &ctx->sync_status) < 0 || ctx->failed) { - mailbox_transaction_rollback(&ctx->t); ctx->failed = TRUE; return -1; } @@ -319,11 +322,6 @@ STATUS_MESSAGES | STATUS_RECENT | STATUS_HIGHESTMODSEQ, &ctx->status); - if (mailbox_transaction_commit(&ctx->t) < 0) { - ctx->failed = TRUE; - ret = -1; - } - if (ctx->status.uidvalidity != client->uidvalidity) { /* most clients would get confused by this. disconnect them. */ client_disconnect_with_error(client, From dovecot at dovecot.org Mon Oct 29 14:19:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:19:59 +0200 Subject: dovecot-2.2: lib-storage: Allow calling mailbox_get_private_flag... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/40e6e734cd5e changeset: 15269:40e6e734cd5e user: Timo Sirainen date: Mon Oct 29 14:19:24 2012 +0200 description: lib-storage: Allow calling mailbox_get_private_flags_mask() without mailbox being open. Most importantly used by mailbox list indexes to determine if it can optimize a STATUS (UNSEEN) call. diffstat: src/lib-storage/mail-storage.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 4e8d7cfec5dc -r 40e6e734cd5e src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Oct 29 14:16:02 2012 +0200 +++ b/src/lib-storage/mail-storage.c Mon Oct 29 14:19:24 2012 +0200 @@ -1447,8 +1447,6 @@ enum mail_flags mailbox_get_private_flags_mask(struct mailbox *box) { - i_assert(box->opened); - if (box->v.get_private_flags_mask != NULL) return box->v.get_private_flags_mask(box); else if (box->list->set.index_pvt_dir != NULL) From dovecot at dovecot.org Mon Oct 29 14:19:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:19:59 +0200 Subject: dovecot-2.2: lib-storage: Mailbox list indexes are now enabled a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/731046f18245 changeset: 15270:731046f18245 user: Timo Sirainen date: Mon Oct 29 14:19:53 2012 +0200 description: lib-storage: Mailbox list indexes are now enabled also for shared/public mailboxes. diffstat: src/lib-storage/list/mailbox-list-index-status.c | 12 +++++++++++- src/lib-storage/list/mailbox-list-index-sync.c | 4 +++- src/lib-storage/list/mailbox-list-index.c | 5 ----- 3 files changed, 14 insertions(+), 7 deletions(-) diffs (61 lines): diff -r 40e6e734cd5e -r 731046f18245 src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Mon Oct 29 14:19:24 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-status.c Mon Oct 29 14:19:53 2012 +0200 @@ -124,6 +124,13 @@ memset(status_r, 0, sizeof(*status_r)); + if ((items & STATUS_UNSEEN) != 0 && + (mailbox_get_private_flags_mask(box) & MAIL_SEEN) != 0) { + /* can't get UNSEEN from list index, since each user has + different \Seen flags */ + return 0; + } + ret = index_list_open_view(box, &view, &seq); if (ret <= 0) return ret; @@ -313,7 +320,10 @@ mailbox_list_index_refresh_later(box->list); else { /* get STATUS info using the given view, rather than - using whatever state the mailbox is currently in */ + using whatever state the mailbox is currently in. + note that for shared mailboxes (with private indexes) this + also means that the unseen count is always the owner's + count, not what exists in the private index. */ hdr = mail_index_get_header(view); memset(&status, 0, sizeof(status)); diff -r 40e6e734cd5e -r 731046f18245 src/lib-storage/list/mailbox-list-index-sync.c --- a/src/lib-storage/list/mailbox-list-index-sync.c Mon Oct 29 14:19:24 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-sync.c Mon Oct 29 14:19:53 2012 +0200 @@ -290,10 +290,12 @@ mailbox_list_index_node_clear_exists(sync_ctx->ilist->mailbox_tree); /* don't include autocreated mailboxes in index until they're - actually created. */ + actually created. this index may be used by multiple users, so + we also want to ignore ACLs here. */ patterns[0] = "*"; patterns[1] = NULL; iter = sync_ctx->ilist->module_ctx.super. iter_init(sync_ctx->list, patterns, + MAILBOX_LIST_ITER_RAW_LIST | MAILBOX_LIST_ITER_NO_AUTO_BOXES); sync_ctx->syncing_list = TRUE; diff -r 40e6e734cd5e -r 731046f18245 src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Mon Oct 29 14:19:24 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index.c Mon Oct 29 14:19:53 2012 +0200 @@ -522,11 +522,6 @@ &dir)) { /* in-memory indexes */ dir = NULL; - } else if (list->ns->type != MAIL_NAMESPACE_TYPE_PRIVATE) { - /* don't create index files for shared/public mailboxes. - their indexes may be shared between multiple users, - each of which may have different ACLs */ - dir = NULL; } i_assert(has_backing_store || dir != NULL); From dovecot at dovecot.org Mon Oct 29 14:31:22 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:31:22 +0200 Subject: dovecot-2.2: Treat modseq updates explicitly instead of as if th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f16da39121a changeset: 15271:0f16da39121a user: Timo Sirainen date: Mon Oct 29 14:31:04 2012 +0200 description: Treat modseq updates explicitly instead of as if they were flag changes. diffstat: src/lib-index/mail-index-view-sync.c | 5 ++++- src/lib-index/mail-index.h | 3 ++- src/lib-storage/index/index-sync.c | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diffs (53 lines): diff -r 731046f18245 -r 0f16da39121a src/lib-index/mail-index-view-sync.c --- a/src/lib-index/mail-index-view-sync.c Mon Oct 29 14:19:53 2012 +0200 +++ b/src/lib-index/mail-index-view-sync.c Mon Oct 29 14:31:04 2012 +0200 @@ -774,7 +774,10 @@ update = CONST_PTR_OFFSET(data, ctx->data_offset); } - rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + if (update->add_flags != 0 || update->remove_flags != 0) + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; + else + rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ; rec->uid1 = update->uid1; rec->uid2 = update->uid2; break; diff -r 731046f18245 -r 0f16da39121a src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Mon Oct 29 14:19:53 2012 +0200 +++ b/src/lib-index/mail-index.h Mon Oct 29 14:31:04 2012 +0200 @@ -185,7 +185,8 @@ enum mail_index_view_sync_type { /* Flags or keywords changed */ - MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS = 0x01 + MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS = 0x01, + MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ = 0x02 }; struct mail_index_view_sync_rec { diff -r 731046f18245 -r 0f16da39121a src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Oct 29 14:19:53 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Mon Oct 29 14:31:04 2012 +0200 @@ -143,6 +143,7 @@ i_array_init(&ctx->hidden_updates, 32); while (mail_index_view_sync_next(ctx->sync_ctx, &sync_rec)) { switch (sync_rec.type) { + case MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ: case MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS: if (!mail_index_lookup_seq_range(ctx->ctx.box->view, sync_rec.uid1, @@ -150,10 +151,11 @@ &seq1, &seq2)) break; - if (!sync_rec.hidden) { + if (!sync_rec.hidden && + sync_rec.type == MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS) { seq_range_array_add_range(&ctx->flag_updates, seq1, seq2); - } else if (array_is_created(&ctx->hidden_updates)) { + } else { seq_range_array_add_range(&ctx->hidden_updates, seq1, seq2); } From dovecot at dovecot.org Mon Oct 29 14:51:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 14:51:04 +0200 Subject: dovecot-2.2: lib-storage: Moved mail attribute dict to struct ma... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ca10d2e8d8e3 changeset: 15272:ca10d2e8d8e3 user: Timo Sirainen date: Mon Oct 29 14:50:11 2012 +0200 description: lib-storage: Moved mail attribute dict to struct mail_storage. This also means that index_storage_destroy() must always be called, so removed now unnecessary mail_storage.destroy=NULL checks. diffstat: src/lib-storage/index/cydir/cydir-storage.c | 2 +- src/lib-storage/index/dbox-common/dbox-storage.c | 1 + src/lib-storage/index/imapc/imapc-storage.c | 1 + src/lib-storage/index/index-attribute.c | 35 ++++++++++----------- src/lib-storage/index/index-storage.c | 12 +++++-- src/lib-storage/index/index-storage.h | 1 + src/lib-storage/index/maildir/maildir-storage.c | 2 +- src/lib-storage/index/mbox/mbox-storage.c | 2 +- src/lib-storage/index/pop3c/pop3c-storage.c | 2 +- src/lib-storage/index/raw/raw-storage.c | 2 +- src/lib-storage/index/shared/shared-storage.c | 2 +- src/lib-storage/mail-storage-private.h | 9 +++-- src/lib-storage/mail-storage.c | 3 +- src/plugins/pop3-migration/pop3-migration-plugin.c | 3 +- src/plugins/virtual/virtual-storage.c | 2 +- 15 files changed, 42 insertions(+), 37 deletions(-) diffs (285 lines): diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/cydir/cydir-storage.c --- a/src/lib-storage/index/cydir/cydir-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/cydir/cydir-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -117,7 +117,7 @@ NULL, cydir_storage_alloc, NULL, - NULL, + index_storage_destroy, NULL, cydir_storage_get_list_settings, NULL, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -122,6 +122,7 @@ if (storage->attachment_fs != NULL) fs_deinit(&storage->attachment_fs); + index_storage_destroy(_storage); } uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list) diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -276,6 +276,7 @@ struct imapc_storage *storage = (struct imapc_storage *)_storage; imapc_client_deinit(&storage->client); + index_storage_destroy(_storage); } static void imapc_storage_add_list(struct mail_storage *_storage, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/index-attribute.c Mon Oct 29 14:50:11 2012 +0200 @@ -4,9 +4,6 @@ #include "dict.h" #include "index-storage.h" -#define KEY_PREFIX_PRIVATE "priv/" -#define KEY_PREFIX_SHARED "shared/" - struct index_storage_attribute_iter { struct mailbox_attribute_iter iter; struct dict_iterate_context *diter; @@ -18,6 +15,7 @@ static int index_storage_get_dict(struct mailbox *box, struct dict **dict_r, const char **mailbox_prefix_r) { + struct mail_storage *storage = box->storage; struct mailbox_metadata metadata; const char *error; @@ -25,31 +23,32 @@ return -1; *mailbox_prefix_r = guid_128_to_string(metadata.guid); - if (box->_attr_dict != NULL) { - *dict_r = box->_attr_dict; + if (storage->_attr_dict != NULL) { + *dict_r = storage->_attr_dict; return 0; } - if (*box->storage->set->mail_attribute_dict == '\0') { - mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, + if (*storage->set->mail_attribute_dict == '\0') { + mail_storage_set_error(storage, MAIL_ERROR_NOTPOSSIBLE, "Mailbox attributes not enabled"); return -1; } - if (box->attr_dict_failed) { - mail_storage_set_internal_error(box->storage); + if (storage->attr_dict_failed) { + mail_storage_set_internal_error(storage); return -1; } - if (dict_init(box->storage->set->mail_attribute_dict, + if (dict_init(storage->set->mail_attribute_dict, DICT_DATA_TYPE_STRING, - box->storage->user->username, - box->storage->user->set->base_dir, - &box->_attr_dict, &error) < 0) { - mail_storage_set_critical(box->storage, + storage->user->username, + storage->user->set->base_dir, + &storage->_attr_dict, &error) < 0) { + mail_storage_set_critical(storage, "mail_attribute_dict: dict_init(%s) failed: %s", - box->storage->set->mail_attribute_dict, error); + storage->set->mail_attribute_dict, error); + storage->attr_dict_failed = TRUE; return -1; } - *dict_r = box->_attr_dict; + *dict_r = storage->_attr_dict; return 0; } @@ -59,10 +58,10 @@ { switch (type) { case MAIL_ATTRIBUTE_TYPE_PRIVATE: - return t_strconcat(KEY_PREFIX_PRIVATE, mailbox_prefix, "/", + return t_strconcat(DICT_PATH_PRIVATE, mailbox_prefix, "/", key, NULL); case MAIL_ATTRIBUTE_TYPE_SHARED: - return t_strconcat(KEY_PREFIX_SHARED, mailbox_prefix, "/", + return t_strconcat(DICT_PATH_SHARED, mailbox_prefix, "/", key, NULL); } i_unreached(); diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/index-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -360,10 +360,6 @@ void index_storage_mailbox_free(struct mailbox *box) { - if (box->_attr_dict != NULL) { - (void)dict_wait(box->_attr_dict); - dict_deinit(&box->_attr_dict); - } if (box->index_pvt != NULL) mail_index_alloc_cache_unref(&box->index_pvt); if (box->index != NULL) @@ -808,3 +804,11 @@ } return 0; } + +void index_storage_destroy(struct mail_storage *storage) +{ + if (storage->_attr_dict != NULL) { + (void)dict_wait(storage->_attr_dict); + dict_deinit(&storage->_attr_dict); + } +} diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/index-storage.h Mon Oct 29 14:50:11 2012 +0200 @@ -155,6 +155,7 @@ void index_copy_cache_fields(struct mail_save_context *ctx, struct mail *src_mail, uint32_t dest_seq); int index_storage_set_subscribed(struct mailbox *box, bool set); +void index_storage_destroy(struct mail_storage *storage); bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1, const ARRAY_TYPE(keyword_indexes) *k2); diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -657,7 +657,7 @@ maildir_get_setting_parser_info, maildir_storage_alloc, maildir_storage_create, - NULL, + index_storage_destroy, maildir_storage_add_list, maildir_storage_get_list_settings, maildir_storage_autodetect, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -793,7 +793,7 @@ mbox_get_setting_parser_info, mbox_storage_alloc, mbox_storage_create, - NULL, + index_storage_destroy, mbox_storage_add_list, mbox_storage_get_list_settings, mbox_storage_autodetect, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/pop3c/pop3c-storage.c --- a/src/lib-storage/index/pop3c/pop3c-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/pop3c/pop3c-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -226,7 +226,7 @@ pop3c_get_setting_parser_info, pop3c_storage_alloc, pop3c_storage_create, - NULL, + index_storage_destroy, NULL, pop3c_storage_get_list_settings, NULL, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/raw/raw-storage.c --- a/src/lib-storage/index/raw/raw-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/raw/raw-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -199,7 +199,7 @@ NULL, raw_storage_alloc, NULL, - NULL, + index_storage_destroy, NULL, raw_storage_get_list_settings, NULL, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/index/shared/shared-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -351,7 +351,7 @@ NULL, shared_storage_alloc, shared_storage_create, - NULL, + index_storage_destroy, NULL, shared_storage_get_list_settings, NULL, diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/mail-storage-private.h Mon Oct 29 14:50:11 2012 +0200 @@ -108,9 +108,14 @@ void *callback_context; struct mail_binary_cache binary_cache; + /* Filled lazily by mailbox_attribute_*() */ + struct dict *_attr_dict; /* Module-specific contexts. See mail_storage_module_id. */ ARRAY(union mail_storage_module_context *) module_contexts; + + /* Failed to create attribute dict, don't try again */ + unsigned int attr_dict_failed:1; }; struct mail_attachment_part { @@ -261,8 +266,6 @@ /* Filled lazily when mailbox is opened, use mailbox_get_path() to access it */ const char *_path; - /* Filled lazily by mailbox_attribute_*() */ - struct dict *_attr_dict; /* default vfuncs for new struct mails. */ const struct mail_vfuncs *mail_vfuncs; @@ -320,8 +323,6 @@ unsigned int disallow_new_keywords:1; /* Mailbox has been synced at least once */ unsigned int synced:1; - /* Failed to create attribute dict, don't try again */ - unsigned int attr_dict_failed:1; }; struct mail_vfuncs { diff -r 0f16da39121a -r ca10d2e8d8e3 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/lib-storage/mail-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -413,8 +413,7 @@ DLLIST_REMOVE(&storage->user->storages, storage); - if (storage->v.destroy != NULL) - storage->v.destroy(storage); + storage->v.destroy(storage); i_free(storage->error_string); *_storage = NULL; diff -r 0f16da39121a -r ca10d2e8d8e3 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Mon Oct 29 14:50:11 2012 +0200 @@ -595,8 +595,7 @@ if (array_is_created(&mstorage->pop3_uidl_map)) array_free(&mstorage->pop3_uidl_map); - if (mstorage->module_ctx.super.destroy != NULL) - mstorage->module_ctx.super.destroy(storage); + mstorage->module_ctx.super.destroy(storage); } static void pop3_migration_mail_storage_created(struct mail_storage *storage) diff -r 0f16da39121a -r ca10d2e8d8e3 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Mon Oct 29 14:31:04 2012 +0200 +++ b/src/plugins/virtual/virtual-storage.c Mon Oct 29 14:50:11 2012 +0200 @@ -503,7 +503,7 @@ NULL, virtual_storage_alloc, NULL, - NULL, + index_storage_destroy, NULL, virtual_storage_get_list_settings, NULL, From dovecot at dovecot.org Mon Oct 29 15:14:02 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:14:02 +0200 Subject: dovecot-2.2: lib-storage: Crashfix to previous stream error chec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f1f2a65d9a1c changeset: 15273:f1f2a65d9a1c user: Timo Sirainen date: Mon Oct 29 15:13:54 2012 +0200 description: lib-storage: Crashfix to previous stream error checking change. diffstat: src/lib-storage/index/index-mail-headers.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (19 lines): diff -r ca10d2e8d8e3 -r f1f2a65d9a1c src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Mon Oct 29 14:50:11 2012 +0200 +++ b/src/lib-storage/index/index-mail-headers.c Mon Oct 29 15:13:54 2012 +0200 @@ -484,8 +484,14 @@ envelope */ message_parse_header(stream, NULL, hdr_parser_flags, imap_envelope_parse_callback, mail); - if (index_mail_stream_check_failure(mail) < 0) + if (stream->stream_errno != 0) { + errno = stream->stream_errno; + mail_storage_set_critical(mail->mail.mail.box->storage, + "read(%s) failed: %m (uid=%u)", + i_stream_get_name(mail->data.stream), + mail->mail.mail.uid); return -1; + } mail->data.save_envelope = FALSE; } From dovecot at dovecot.org Mon Oct 29 15:17:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:17:23 +0200 Subject: dovecot-2.2: lib-index: Fixes to replacing message's keywords. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a87edad22199 changeset: 15274:a87edad22199 user: Timo Sirainen date: Mon Oct 29 15:17:17 2012 +0200 description: lib-index: Fixes to replacing message's keywords. diffstat: src/lib-index/mail-index-transaction.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (24 lines): diff -r f1f2a65d9a1c -r a87edad22199 src/lib-index/mail-index-transaction.c --- a/src/lib-index/mail-index-transaction.c Mon Oct 29 15:13:54 2012 +0200 +++ b/src/lib-index/mail-index-transaction.c Mon Oct 29 15:17:17 2012 +0200 @@ -97,13 +97,19 @@ { uint32_t uid, latest_seq; + /* seq points to the transaction's primary view */ + mail_index_lookup_uid(t->view, seq, &uid); + + /* get the latest keywords from the updated index, or fallback to the + primary view if the message is already expunged */ if (t->latest_view == NULL) { mail_index_refresh(t->view->index); t->latest_view = mail_index_view_open(t->view->index); } - mail_index_lookup_uid(t->latest_view, seq, &uid); if (mail_index_lookup_seq(t->latest_view, uid, &latest_seq)) mail_index_lookup_keywords(t->latest_view, latest_seq, keywords); + else + mail_index_lookup_keywords(t->view, seq, keywords); } static int From dovecot at dovecot.org Mon Oct 29 15:19:11 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:19:11 +0200 Subject: dovecot-2.2: lib-storage: Fixes to handling separation of privat... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c84efce3a94 changeset: 15275:4c84efce3a94 user: Timo Sirainen date: Mon Oct 29 15:18:34 2012 +0200 description: lib-storage: Fixes to handling separation of private/shared attributes. diffstat: src/lib-storage/index/index-attribute.c | 102 ++++++++++++++++++++++++++++--- src/lib-storage/index/index-storage.c | 6 +- src/lib-storage/mail-storage-private.h | 9 +- src/lib-storage/mail-user.c | 5 + src/lib-storage/mail-user.h | 4 + 5 files changed, 108 insertions(+), 18 deletions(-) diffs (242 lines): diff -r a87edad22199 -r 4c84efce3a94 src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Mon Oct 29 15:17:17 2012 +0200 +++ b/src/lib-storage/index/index-attribute.c Mon Oct 29 15:18:34 2012 +0200 @@ -12,10 +12,74 @@ bool dict_disabled; }; -static int index_storage_get_dict(struct mailbox *box, struct dict **dict_r, - const char **mailbox_prefix_r) +static struct mail_namespace * +mail_user_find_attribute_namespace(struct mail_user *user) +{ + struct mail_namespace *ns; + + ns = mail_namespace_find_inbox(user->namespaces); + if (ns != NULL) + return ns; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->type == MAIL_NAMESPACE_TYPE_PRIVATE) + return ns; + } + return NULL; +} + +static int +index_storage_get_user_dict(struct mail_storage *err_storage, + struct mail_user *user, struct dict **dict_r) +{ + struct mail_namespace *ns; + struct mail_storage *attr_storage; + const char *error; + + if (user->_attr_dict != NULL) { + *dict_r = user->_attr_dict; + return 0; + } + if (user->attr_dict_failed) { + mail_storage_set_internal_error(err_storage); + return -1; + } + + ns = mail_user_find_attribute_namespace(user); + if (ns == NULL) { + /* probably never happens? */ + mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not available for this mailbox"); + return -1; + } + attr_storage = mail_namespace_get_default_storage(ns); + + if (*attr_storage->set->mail_attribute_dict == '\0') { + mail_storage_set_error(err_storage, MAIL_ERROR_NOTPOSSIBLE, + "Mailbox attributes not enabled"); + return -1; + } + + if (dict_init(attr_storage->set->mail_attribute_dict, + DICT_DATA_TYPE_STRING, + user->username, user->set->base_dir, + &user->_attr_dict, &error) < 0) { + mail_storage_set_critical(err_storage, + "mail_attribute_dict: dict_init(%s) failed: %s", + attr_storage->set->mail_attribute_dict, error); + user->attr_dict_failed = TRUE; + return -1; + } + *dict_r = user->_attr_dict; + return 0; +} + +static int +index_storage_get_dict(struct mailbox *box, enum mail_attribute_type type, + struct dict **dict_r, const char **mailbox_prefix_r) { struct mail_storage *storage = box->storage; + struct mail_namespace *ns; struct mailbox_metadata metadata; const char *error; @@ -23,8 +87,24 @@ return -1; *mailbox_prefix_r = guid_128_to_string(metadata.guid); - if (storage->_attr_dict != NULL) { - *dict_r = storage->_attr_dict; + ns = mailbox_get_namespace(box); + if (type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + /* private attributes are stored in user's own dict */ + return index_storage_get_user_dict(storage, storage->user, dict_r); + } else if (ns->user == ns->owner) { + /* user owns the mailbox. shared attributes are stored in + the same dict. */ + return index_storage_get_user_dict(storage, storage->user, dict_r); + } else if (ns->owner != NULL) { + /* accessing shared attribute of a shared mailbox. + use the owner's dict. */ + return index_storage_get_user_dict(storage, ns->owner, dict_r); + } + + /* accessing shared attributes of a public mailbox. no user owns it, + so use the storage's dict. */ + if (storage->_shared_attr_dict != NULL) { + *dict_r = storage->_shared_attr_dict; return 0; } if (*storage->set->mail_attribute_dict == '\0') { @@ -32,7 +112,7 @@ "Mailbox attributes not enabled"); return -1; } - if (storage->attr_dict_failed) { + if (storage->shared_attr_dict_failed) { mail_storage_set_internal_error(storage); return -1; } @@ -41,14 +121,14 @@ DICT_DATA_TYPE_STRING, storage->user->username, storage->user->set->base_dir, - &storage->_attr_dict, &error) < 0) { + &storage->_shared_attr_dict, &error) < 0) { mail_storage_set_critical(storage, "mail_attribute_dict: dict_init(%s) failed: %s", storage->set->mail_attribute_dict, error); - storage->attr_dict_failed = TRUE; + storage->shared_attr_dict_failed = TRUE; return -1; } - *dict_r = storage->_attr_dict; + *dict_r = storage->_shared_attr_dict; return 0; } @@ -75,7 +155,7 @@ struct dict *dict; const char *mailbox_prefix; - if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) + if (index_storage_get_dict(box, type, &dict, &mailbox_prefix) < 0) return -1; T_BEGIN { @@ -101,7 +181,7 @@ const char *mailbox_prefix; int ret; - if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) + if (index_storage_get_dict(box, type, &dict, &mailbox_prefix) < 0) return -1; ret = dict_lookup(dict, pool_datastack_create(), @@ -126,7 +206,7 @@ iter = i_new(struct index_storage_attribute_iter, 1); iter->iter.box = box; - if (index_storage_get_dict(box, &dict, &mailbox_prefix) < 0) { + if (index_storage_get_dict(box, type, &dict, &mailbox_prefix) < 0) { if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTPOSSIBLE) iter->dict_disabled = TRUE; } else { diff -r a87edad22199 -r 4c84efce3a94 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Mon Oct 29 15:17:17 2012 +0200 +++ b/src/lib-storage/index/index-storage.c Mon Oct 29 15:18:34 2012 +0200 @@ -807,8 +807,8 @@ void index_storage_destroy(struct mail_storage *storage) { - if (storage->_attr_dict != NULL) { - (void)dict_wait(storage->_attr_dict); - dict_deinit(&storage->_attr_dict); + if (storage->_shared_attr_dict != NULL) { + (void)dict_wait(storage->_shared_attr_dict); + dict_deinit(&storage->_shared_attr_dict); } } diff -r a87edad22199 -r 4c84efce3a94 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Oct 29 15:17:17 2012 +0200 +++ b/src/lib-storage/mail-storage-private.h Mon Oct 29 15:18:34 2012 +0200 @@ -108,14 +108,15 @@ void *callback_context; struct mail_binary_cache binary_cache; - /* Filled lazily by mailbox_attribute_*() */ - struct dict *_attr_dict; + /* Filled lazily by mailbox_attribute_*() when accessing shared + attributes. */ + struct dict *_shared_attr_dict; /* Module-specific contexts. See mail_storage_module_id. */ ARRAY(union mail_storage_module_context *) module_contexts; - /* Failed to create attribute dict, don't try again */ - unsigned int attr_dict_failed:1; + /* Failed to create shared attribute dict, don't try again */ + unsigned int shared_attr_dict_failed:1; }; struct mail_attachment_part { diff -r a87edad22199 -r 4c84efce3a94 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Mon Oct 29 15:17:17 2012 +0200 +++ b/src/lib-storage/mail-user.c Mon Oct 29 15:18:34 2012 +0200 @@ -14,6 +14,7 @@ #include "auth-master.h" #include "master-service.h" #include "mountpoint-list.h" +#include "dict.h" #include "mail-storage-settings.h" #include "mail-storage-private.h" #include "mail-storage-service.h" @@ -28,6 +29,10 @@ static void mail_user_deinit_base(struct mail_user *user) { + if (user->_attr_dict != NULL) { + (void)dict_wait(user->_attr_dict); + dict_deinit(&user->_attr_dict); + } mail_namespaces_deinit(&user->namespaces); if (user->mountpoints != NULL) mountpoint_list_deinit(&user->mountpoints); diff -r a87edad22199 -r 4c84efce3a94 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Mon Oct 29 15:17:17 2012 +0200 +++ b/src/lib-storage/mail-user.h Mon Oct 29 15:18:34 2012 +0200 @@ -40,6 +40,8 @@ struct mountpoint_list *mountpoints; normalizer_func_t *default_normalizer; + /* Filled lazily by mailbox_attribute_*() when accessing attributes. */ + struct dict *_attr_dict; /* Module-specific contexts. See mail_storage_module_id. */ ARRAY(union mail_user_module_context *) module_contexts; @@ -64,6 +66,8 @@ unsigned int fuzzy_search:1; /* We're running dsync */ unsigned int dsyncing:1; + /* Failed to create attribute dict, don't try again */ + unsigned int attr_dict_failed:1; }; struct mail_user_module_register { From dovecot at dovecot.org Mon Oct 29 15:31:15 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:31:15 +0200 Subject: dovecot-2.2: lib-storage: Added struct mail_user.nonexistent fla... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5d465a3eceb2 changeset: 15276:5d465a3eceb2 user: Timo Sirainen date: Mon Oct 29 15:29:52 2012 +0200 description: lib-storage: Added struct mail_user.nonexistent flag, which is filled by userdb lookup. diffstat: src/lib-storage/mail-user.c | 19 +++++++++++-------- src/lib-storage/mail-user.h | 3 +++ 2 files changed, 14 insertions(+), 8 deletions(-) diffs (50 lines): diff -r 4c84efce3a94 -r 5d465a3eceb2 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Mon Oct 29 15:18:34 2012 +0200 +++ b/src/lib-storage/mail-user.c Mon Oct 29 15:29:52 2012 +0200 @@ -296,9 +296,6 @@ if (user->remote_ip != NULL) info.remote_ip = *user->remote_ip; - if (mail_user_auth_master_conn == NULL) - return 0; - userdb_pool = pool_alloconly_create("userdb lookup", 2048); ret = auth_master_user_lookup(mail_user_auth_master_conn, user->username, &info, userdb_pool, @@ -320,12 +317,18 @@ return user->_home != NULL ? 1 : 0; } - ret = mail_user_userdb_lookup_home(user); - if (ret < 0) + if (mail_user_auth_master_conn == NULL) { + /* no userdb connection. we can only use mail_home setting. */ + user->_home = user->set->mail_home; + } else if ((ret = mail_user_userdb_lookup_home(user)) < 0) { + /* userdb lookup failed */ return -1; - - if (ret > 0 && user->_home == NULL && *user->set->mail_home != '\0') { - /* no home in userdb, fallback to mail_home setting */ + } else if (ret == 0) { + /* user doesn't exist */ + user->nonexistent = TRUE; + } else if (user->_home == NULL && *user->set->mail_home != '\0') { + /* no home returned by userdb lookup, fallback to mail_home + setting. */ user->_home = user->set->mail_home; } user->home_looked_up = TRUE; diff -r 4c84efce3a94 -r 5d465a3eceb2 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Mon Oct 29 15:18:34 2012 +0200 +++ b/src/lib-storage/mail-user.h Mon Oct 29 15:29:52 2012 +0200 @@ -46,6 +46,9 @@ /* Module-specific contexts. See mail_storage_module_id. */ ARRAY(union mail_user_module_context *) module_contexts; + /* User doesn't exist (as reported by userdb lookup when looking + up home) */ + unsigned int nonexistent:1; /* Either home is set or there is no home for the user. */ unsigned int home_looked_up:1; /* User is an administrator. Allow operations not normally allowed From dovecot at dovecot.org Mon Oct 29 15:31:15 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:31:15 +0200 Subject: dovecot-2.2: lib-storage: Handle better when attempting to acces... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a6b21ce8652d changeset: 15277:a6b21ce8652d user: Timo Sirainen date: Mon Oct 29 15:30:59 2012 +0200 description: lib-storage: Handle better when attempting to access shared mailboxes for nonexistent users. diffstat: src/lib-storage/index/shared/shared-storage.c | 18 ++++++++++-------- 1 files changed, 10 insertions(+), 8 deletions(-) diffs (29 lines): diff -r 5d465a3eceb2 -r a6b21ce8652d src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Mon Oct 29 15:29:52 2012 +0200 +++ b/src/lib-storage/index/shared/shared-storage.c Mon Oct 29 15:30:59 2012 +0200 @@ -257,15 +257,17 @@ user->unexpanded_set); owner->autocreated = TRUE; if (mail_user_init(owner, &error) < 0) { - mailbox_list_set_critical(list, - "Couldn't create namespace '%s' for user %s: %s", - ns->prefix, userdomain, error); - mail_user_unref(&owner); - return -1; - } - if (!var_has_key(storage->location, 'h', "home")) + if (!owner->nonexistent) { + mailbox_list_set_critical(list, + "Couldn't create namespace '%s' for user %s: %s", + ns->prefix, userdomain, error); + mail_user_unref(&owner); + return -1; + } + ret = 0; + } else if (!var_has_key(storage->location, 'h', "home")) { ret = 1; - else { + } else { /* we'll need to look up the user's home directory */ if ((ret = mail_user_get_home(owner, &tab[3].value)) < 0) { mailbox_list_set_critical(list, "Namespace '%s': " From dovecot at dovecot.org Mon Oct 29 15:32:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 15:32:54 +0200 Subject: dovecot-2.2: TODO updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12cd194553a8 changeset: 15278:12cd194553a8 user: Timo Sirainen date: Mon Oct 29 15:32:44 2012 +0200 description: TODO updated diffstat: TODO | 21 +++++++++++++-------- 1 files changed, 13 insertions(+), 8 deletions(-) diffs (44 lines): diff -r a6b21ce8652d -r 12cd194553a8 TODO --- a/TODO Mon Oct 29 15:30:59 2012 +0200 +++ b/TODO Mon Oct 29 15:32:44 2012 +0200 @@ -1,19 +1,25 @@ - - libssl-iostream read all of file input stream, no buffer limits in ssl - - finish dsync rewrite + - LAYOUT=index: + - force-resync should fix the index, finding any missing mailboxes, same + for when internally detecting the error + - see if there are any race conditions? and check other error handling + - after doing a lot of changes the list's memory pool keeps growing. + do an occasional re-parsing to clear the pool + - mailbox_update() needs to update also uidvalidity/guid + - quota recalc + dict-file [+acl?] assert-crashes in !indexing->syncing + - URLAUTH: if client tries to access nonexistent user, do a delay in + imap-urlauth-client.c (AFTER destroying the worker) + - special response in the control connection to make the imap-urlauth + master wait before starting a new worker + - settings parsing is horribly bloaty - doveadm: if running via doveadm-server and it fails, say something about error being in the log - indexer-worker and maybe others (doveadm?) could support dropping privileges permanently when service_count=1. Note that LMTP can't with multiple RCPT TOs. - after reading whole message text, update has_nul-state to cache - - if indexpvt is enabled, mailbox_list_indexes should go there? at least - private flags are otherwise problematic.. possibly only for shared/public - mailboxes?.. - - index_mail_parse_headers() etc. message_parsers don't check for stream errors - FIFOs maybe should be counted as connections, but unlisten should unlink+reopen it in master? - - change proxy TTL so it stops at 1? (instead of 0) - lmtp client/proxy: Handle multiline replies better - recreate mailbox -> existing sessions log "indexid changed" error - add message/mime limits @@ -125,7 +131,6 @@ - ldap: fix multiple-gid support somehow - search: use mail_get_parts() only when it's already cached. if it's not, add it to cache afterwards. - - move ssl proxying code to lib-master - dict pooling /* currently non-external transactions can be applied multiple times, From dovecot at dovecot.org Mon Oct 29 16:37:21 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 16:37:21 +0200 Subject: dovecot-2.1: lib-dict: Abort async transaction commits if client... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/67e9cb0b06ec changeset: 14790:67e9cb0b06ec user: Timo Sirainen date: Mon Oct 29 16:36:59 2012 +0200 description: lib-dict: Abort async transaction commits if client gets disconnected from dict server. diffstat: src/lib-dict/dict-client.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (38 lines): diff -r 22875bcaa952 -r 67e9cb0b06ec src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Fri Oct 26 13:05:43 2012 +0300 +++ b/src/lib-dict/dict-client.c Mon Oct 29 16:36:59 2012 +0200 @@ -72,6 +72,7 @@ unsigned int failed:1; unsigned int sent_begin:1; + unsigned int async:1; }; static int client_dict_connect(struct client_dict *dict); @@ -444,9 +445,18 @@ static void client_dict_disconnect(struct client_dict *dict) { + struct client_dict_transaction_context *ctx, *next; + dict->connect_counter++; dict->handshaked = FALSE; + /* abort all pending async commits */ + for (ctx = dict->transactions; ctx != NULL; ctx = next) { + next = ctx->next; + if (ctx->async) + client_dict_finish_transaction(dict, ctx->id, -1); + } + if (dict->to_idle != NULL) timeout_remove(&dict->to_idle); if (dict->io != NULL) @@ -706,6 +716,7 @@ else if (async) { ctx->callback = callback; ctx->context = context; + ctx->async = TRUE; if (dict->async_commits++ == 0) { dict->io = io_add(dict->fd, IO_READ, dict_async_input, dict); From dovecot at dovecot.org Mon Oct 29 16:51:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 16:51:56 +0200 Subject: dovecot-2.2: lib-fs: Compile fix for OSes that don't support pos... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0b4b2e37b793 changeset: 15279:0b4b2e37b793 user: Timo Sirainen date: Mon Oct 29 16:51:46 2012 +0200 description: lib-fs: Compile fix for OSes that don't support posix_fadvise(POSIX_FADV_WILLNEED) diffstat: src/lib-fs/fs-posix.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 12cd194553a8 -r 0b4b2e37b793 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Mon Oct 29 15:32:44 2012 +0200 +++ b/src/lib-fs/fs-posix.c Mon Oct 29 16:51:46 2012 +0200 @@ -286,7 +286,7 @@ i_free(file); } -static bool fs_posix_prefetch(struct fs_file *_file, uoff_t length) +static bool fs_posix_prefetch(struct fs_file *_file, uoff_t length ATTR_UNUSED) { struct posix_fs_file *file = (struct posix_fs_file *)_file; @@ -295,10 +295,13 @@ return TRUE; } +/* HAVE_POSIX_FADVISE alone isn't enough for CentOS 4.9 */ +#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_WILLNEED) if (posix_fadvise(file->fd, 0, length, POSIX_FADV_WILLNEED) < 0) { i_error("posix_fadvise(%s) failed: %m", _file->path); return TRUE; } +#endif return FALSE; } From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lazy-expunge: Fixed handling non-default namespace ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e431b202cfd changeset: 15280:4e431b202cfd user: Timo Sirainen date: Wed Sep 26 21:12:37 2012 +0300 description: lazy-expunge: Fixed handling non-default namespace separator. diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (27 lines): diff -r 4d82c74f702c -r 4e431b202cfd src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Mon Sep 24 17:03:42 2012 +0300 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Wed Sep 26 21:12:37 2012 +0300 @@ -70,11 +70,11 @@ const char *name; char src_sep, dest_sep; - /* get the storage name, so it doesn't have namespace prefix */ + /* use the (canonical / unaliased) storage name */ name = src_box->name; - /* replace hierarchy separators with destination separator */ + /* replace hierarchy separators with destination virtual separator */ src_sep = mailbox_list_get_hierarchy_sep(src_box->list); - dest_sep = mailbox_list_get_hierarchy_sep(list); + dest_sep = mail_namespace_get_sep(list->ns); if (src_sep != dest_sep) { string_t *str = t_str_new(128); unsigned int i; @@ -87,7 +87,7 @@ } name = str_c(str); } - /* add expunge namespace prefix */ + /* add expunge namespace prefix. the name is now a proper vname */ name = t_strconcat(list->ns->prefix, name, NULL); box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES); From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: i_getpwnam(): Ignore EINVAL errors silently. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9dcc44d8275d changeset: 15281:9dcc44d8275d user: Timo Sirainen date: Wed Sep 26 21:14:23 2012 +0300 description: i_getpwnam(): Ignore EINVAL errors silently. At least FreeBSD returns it when attempting to lookup user at domain. diffstat: src/lib/ipwd.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 4e431b202cfd -r 9dcc44d8275d src/lib/ipwd.c --- a/src/lib/ipwd.c Wed Sep 26 21:12:37 2012 +0300 +++ b/src/lib/ipwd.c Wed Sep 26 21:14:23 2012 +0300 @@ -58,6 +58,10 @@ errno = getpwnam_r(name, pwd_r, pwbuf, pwbuf_size, &result); if (result != NULL) return 1; + if (errno == EINVAL) { + /* FreeBSD fails here when name="user at domain" */ + return 0; + } return errno == 0 ? 0 : -1; } From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: doveadm altmove: Make sure all storages get purged ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4819306a1f9f changeset: 15282:4819306a1f9f user: Timo Sirainen date: Thu Sep 27 00:19:41 2012 +0300 description: doveadm altmove: Make sure all storages get purged (and not more than once) diffstat: src/doveadm/doveadm-mail-altmove.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (24 lines): diff -r 9dcc44d8275d -r 4819306a1f9f src/doveadm/doveadm-mail-altmove.c --- a/src/doveadm/doveadm-mail-altmove.c Wed Sep 26 21:14:23 2012 +0300 +++ b/src/doveadm/doveadm-mail-altmove.c Thu Sep 27 00:19:41 2012 +0300 @@ -87,6 +87,12 @@ if (doveadm_mailbox_list_iter_deinit(&iter) < 0) ret = -1; + if (prev_ns != NULL) { + if (ns_purge(_ctx, prev_ns) < 0) + ret = -1; + array_append(&purged_storages, &prev_ns->storage, 1); + } + /* make sure all private storages have been purged */ storages = array_get(&purged_storages, &count); for (ns = user->namespaces; ns != NULL; ns = ns->next) { @@ -101,6 +107,7 @@ if (ns_purge(_ctx, ns) < 0) ret = -1; array_append(&purged_storages, &ns->storage, 1); + storages = array_get(&purged_storages, &count); } } return ret; From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: mdbox: Don't crash in storage rebuild if mail's "or... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/88a05f387743 changeset: 15283:88a05f387743 user: Timo Sirainen date: Thu Sep 27 02:55:14 2012 +0300 description: mdbox: Don't crash in storage rebuild if mail's "original mailbox" metadata is missing. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (16 lines): diff -r 4819306a1f9f -r 88a05f387743 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Sep 27 00:19:41 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Sep 27 02:55:14 2012 +0300 @@ -605,8 +605,10 @@ if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { mailbox = dbox_file_metadata_get(file, DBOX_METADATA_ORIG_MAILBOX); - mailbox = mailbox_list_get_vname(ctx->default_list, mailbox); - mailbox = t_strdup(mailbox); + if (mailbox != NULL) { + mailbox = mailbox_list_get_vname(ctx->default_list, mailbox); + mailbox = t_strdup(mailbox); + } } dbox_file_unref(&file); if (ret <= 0 || deleted) { From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: Avoid using PATH_MAX. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/75aadea5c2a2 changeset: 15284:75aadea5c2a2 user: Timo Sirainen date: Fri Sep 28 15:07:11 2012 +0300 description: Avoid using PATH_MAX. diffstat: src/lib-master/master-instance.c | 2 +- src/lib-storage/index/dbox-common/dbox-storage.c | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diffs (46 lines): diff -r 88a05f387743 -r 75aadea5c2a2 src/lib-master/master-instance.c --- a/src/lib-master/master-instance.c Thu Sep 27 02:55:14 2012 +0300 +++ b/src/lib-master/master-instance.c Fri Sep 28 15:07:11 2012 +0300 @@ -110,7 +110,7 @@ i_error("open(%s) failed: %m", list->path); return -1; } - input = i_stream_create_fd(fd, PATH_MAX, TRUE); + input = i_stream_create_fd(fd, (size_t)-1, TRUE); while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN { if (master_instance_list_add_line(list, line) < 0) i_error("Invalid line in %s: %s", list->path, line); diff -r 88a05f387743 -r 75aadea5c2a2 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Thu Sep 27 02:55:14 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Fri Sep 28 15:07:11 2012 +0300 @@ -31,25 +31,23 @@ dbox_alt_path_has_changed(const char *root_dir, const char *alt_path, const char *alt_symlink_path) { - char buf[PATH_MAX]; + const char *linkpath; ssize_t ret; - ret = readlink(alt_symlink_path, buf, sizeof(buf)-1); - if (ret < 0) { + if (t_readlink(alt_symlink_path, &linkpath) < 0) { if (errno == ENOENT) return alt_path != NULL; i_error("readlink(%s) failed: %m", alt_symlink_path); return FALSE; } - buf[ret] = '\0'; if (alt_path == NULL) { i_warning("dbox %s: Original ALT=%s, " - "but currently no ALT path set", root_dir, buf); + "but currently no ALT path set", root_dir, linkpath); return TRUE; - } else if (strcmp(buf, alt_path) != 0) { + } else if (strcmp(linkpath, alt_path) != 0) { i_warning("dbox %s: Original ALT=%s, " - "but currently ALT=%s", root_dir, buf, alt_path); + "but currently ALT=%s", root_dir, linkpath, alt_path); return TRUE; } return FALSE; From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: Avoid using PATH_MAX. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2a44991cbf66 changeset: 15285:2a44991cbf66 user: Timo Sirainen date: Fri Sep 28 15:11:30 2012 +0300 description: Avoid using PATH_MAX. diffstat: src/lib-storage/index/dbox-common/dbox-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (19 lines): diff -r 75aadea5c2a2 -r 2a44991cbf66 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Fri Sep 28 15:07:11 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Fri Sep 28 15:11:30 2012 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "abspath.h" #include "ioloop.h" #include "fs-api.h" #include "mkdir-parents.h" @@ -32,7 +33,6 @@ const char *alt_path, const char *alt_symlink_path) { const char *linkpath; - ssize_t ret; if (t_readlink(alt_symlink_path, &linkpath) < 0) { if (errno == ENOENT) From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: Compiling fix for HURD Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6cac808c4bd8 changeset: 15286:6cac808c4bd8 user: Timo Sirainen date: Fri Sep 28 15:12:28 2012 +0300 description: Compiling fix for HURD diffstat: src/lib/compat.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 2a44991cbf66 -r 6cac808c4bd8 src/lib/compat.h --- a/src/lib/compat.h Fri Sep 28 15:11:30 2012 +0300 +++ b/src/lib/compat.h Fri Sep 28 15:12:28 2012 +0300 @@ -270,4 +270,8 @@ # define IO_BLOCK_SIZE 8192 #endif +#if !defined(PIPE_BUF) && defined(_POSIX_PIPE_BUF) +# define PIPE_BUF (8 * _POSIX_PIPE_BUF) /* for HURD */ #endif + +#endif From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lmtp: Fixed hanging on proxying if remote server wa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/38727d3e90ec changeset: 15287:38727d3e90ec user: Timo Sirainen date: Tue Oct 02 21:36:43 2012 +0300 description: lmtp: Fixed hanging on proxying if remote server was down. Patch by Jack Bates. diffstat: src/lmtp/lmtp-proxy.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (10 lines): diff -r 6cac808c4bd8 -r 38727d3e90ec src/lmtp/lmtp-proxy.c --- a/src/lmtp/lmtp-proxy.c Fri Sep 28 15:12:28 2012 +0300 +++ b/src/lmtp/lmtp-proxy.c Tue Oct 02 21:36:43 2012 +0300 @@ -300,4 +300,6 @@ lmtp_client_send(conn->client, conn->data_input); lmtp_client_send_more(conn->client); } + /* finish if all of the connections have already failed */ + lmtp_proxy_try_finish(proxy); } From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: mbox: Fixed getting filesystem permissions when par... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/83695d6d41aa changeset: 15288:83695d6d41aa user: Timo Sirainen date: Tue Oct 02 21:56:09 2012 +0300 description: mbox: Fixed getting filesystem permissions when parent dir has setgid-bit enabled. diffstat: src/lib-storage/mailbox-list.c | 17 ++++++++++++++++- 1 files changed, 16 insertions(+), 1 deletions(-) diffs (34 lines): diff -r 38727d3e90ec -r 83695d6d41aa src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue Oct 02 21:36:43 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Tue Oct 02 21:56:09 2012 +0300 @@ -638,7 +638,7 @@ void mailbox_list_get_permissions(struct mailbox_list *list, const char *name, struct mailbox_permissions *permissions_r) { - const char *path, *parent_name, *p; + const char *path, *parent_name, *parent_path, *p; struct stat st; memset(permissions_r, 0, sizeof(*permissions_r)); @@ -708,6 +708,21 @@ } else { permissions_r->file_create_gid = st.st_gid; } + if (!S_ISDIR(st.st_mode) && + permissions_r->file_create_gid != (gid_t)-1) { + /* we need to stat() the parent directory to see if + it has setgid-bit set */ + p = strrchr(path, '/'); + parent_path = p == NULL ? NULL : + t_strdup_until(path, p); + if (parent_path != NULL && + stat(parent_path, &st) == 0 && + (st.st_mode & S_ISGID) != 0) { + /* directory's GID is used automatically for + new files */ + permissions_r->file_create_gid = (gid_t)-1; + } + } } if (name == NULL) { From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: doveadm: Fixed printing large input from doveadm-se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/94c7e875f9b9 changeset: 15289:94c7e875f9b9 user: Timo Sirainen date: Tue Oct 02 22:37:49 2012 +0300 description: doveadm: Fixed printing large input from doveadm-server. diffstat: src/doveadm/server-connection.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 83695d6d41aa -r 94c7e875f9b9 src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Tue Oct 02 21:56:09 2012 +0300 +++ b/src/doveadm/server-connection.c Tue Oct 02 22:37:49 2012 +0300 @@ -100,7 +100,8 @@ { if (conn->streaming) { conn->streaming = FALSE; - stream_data(str, data, size); + if (size > 0) + stream_data(str, data, size); doveadm_print_stream("", 0); } else { const char *text; From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lib-master: Fixed crashes with settings cache. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e29b627219b3 changeset: 15290:e29b627219b3 user: Timo Sirainen date: Tue Oct 02 23:12:07 2012 +0300 description: lib-master: Fixed crashes with settings cache. diffstat: src/lib-master/master-service-settings-cache.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 94c7e875f9b9 -r e29b627219b3 src/lib-master/master-service-settings-cache.c --- a/src/lib-master/master-service-settings-cache.c Tue Oct 02 22:37:49 2012 +0300 +++ b/src/lib-master/master-service-settings-cache.c Tue Oct 02 23:12:07 2012 +0300 @@ -83,6 +83,7 @@ } for (entry = cache->oldest; entry != NULL; entry = next) { next = entry->next; + i_assert(entry->parser != cache->global_parser); settings_parser_deinit(&entry->parser); pool_unref(&entry->pool); } @@ -146,8 +147,10 @@ } if (entry != NULL) { - DLLIST2_REMOVE(&cache->oldest, &cache->newest, entry); - DLLIST2_APPEND(&cache->oldest, &cache->newest, entry); + if (entry->parser != cache->global_parser) { + DLLIST2_REMOVE(&cache->oldest, &cache->newest, entry); + DLLIST2_APPEND(&cache->oldest, &cache->newest, entry); + } *parser_r = entry->parser; return TRUE; } From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lib-storage: When index mkdir() fails with EPERM, c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ce50c0bb782 changeset: 15291:3ce50c0bb782 user: Timo Sirainen date: Tue Oct 02 23:24:10 2012 +0300 description: lib-storage: When index mkdir() fails with EPERM, create the dir anyway with 0700 mode. This avoids failing entirely when /var/mail/user has 0660 permissions and we don't have access to the group. The error message is still logged. diffstat: src/lib-storage/mailbox-list.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (19 lines): diff -r e29b627219b3 -r 3ce50c0bb782 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Tue Oct 02 23:12:07 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Tue Oct 02 23:24:10 2012 +0300 @@ -1461,7 +1461,14 @@ if (errno != ENOENT || p == NULL || ++n == 2) { mailbox_list_set_critical(list, "mkdir(%s) failed: %m", index_dir); - return -1; + if (p == NULL || errno != EPERM || + perm.dir_create_mode == 0700) + return -1; + /* we can't use the GID. allow it anyway with more + restricted permissions. */ + perm.file_create_gid = (gid_t)-1; + perm.dir_create_mode = 0700; + continue; } /* create the parent directory first */ parent_dir = t_strdup_until(index_dir, p); From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: dbox: Small code cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8c06fe0b280d changeset: 15292:8c06fe0b280d user: Timo Sirainen date: Wed Oct 03 01:12:13 2012 +0300 description: dbox: Small code cleanup. diffstat: src/lib-storage/index/dbox-common/dbox-save.c | 15 ++++++++------- src/lib-storage/index/dbox-multi/mdbox-save.c | 2 -- src/lib-storage/index/dbox-single/sdbox-save.c | 6 +----- 3 files changed, 9 insertions(+), 14 deletions(-) diffs (55 lines): diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-common/dbox-save.c --- a/src/lib-storage/index/dbox-common/dbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -105,13 +105,14 @@ if (index_attachment_save_finish(&ctx->ctx) < 0) ctx->failed = TRUE; } - if (ctx->ctx.output == dbox_output) - return; - - /* e.g. zlib plugin had changed this */ - o_stream_ref(dbox_output); - o_stream_destroy(&ctx->ctx.output); - ctx->ctx.output = dbox_output; + if (ctx->ctx.output != dbox_output) { + /* e.g. zlib plugin had changed this */ + o_stream_ref(dbox_output); + o_stream_destroy(&ctx->ctx.output); + ctx->ctx.output = dbox_output; + } + index_mail_cache_parse_deinit(ctx->ctx.dest_mail, + ctx->ctx.received_date, !ctx->failed); } void dbox_save_write_metadata(struct mail_save_context *_ctx, diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -199,8 +199,6 @@ return -1; dbox_save_end(&ctx->ctx); - index_mail_cache_parse_deinit(_ctx->dest_mail, - _ctx->received_date, !ctx->ctx.failed); mail = array_idx_modifiable(&ctx->mails, array_count(&ctx->mails) - 1); if (!ctx->ctx.failed) T_BEGIN { diff -r 3ce50c0bb782 -r 8c06fe0b280d src/lib-storage/index/dbox-single/sdbox-save.c --- a/src/lib-storage/index/dbox-single/sdbox-save.c Tue Oct 02 23:24:10 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:12:13 2012 +0300 @@ -181,13 +181,9 @@ index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t)); } - - index_mail_cache_parse_deinit(_ctx->dest_mail, - _ctx->received_date, !ctx->ctx.failed); + dbox_save_end(&ctx->ctx); files = array_idx_modifiable(&ctx->files, array_count(&ctx->files) - 1); - - dbox_save_end(&ctx->ctx); if (!ctx->ctx.failed) T_BEGIN { if (dbox_save_mail_write_metadata(&ctx->ctx, *files) < 0) ctx->ctx.failed = TRUE; From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lib-storage: mailbox_save_cancel() now makes sure t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3de5a5b49580 changeset: 15294:3de5a5b49580 user: Timo Sirainen date: Wed Oct 03 01:20:22 2012 +0300 description: lib-storage: mailbox_save_cancel() now makes sure that dest_mail is reset. This fixes e.g. doveadm import, which continues import even though some messages couldn't be saved. diffstat: src/lib-storage/mail-storage.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (23 lines): diff -r 4c3dac1a94cf -r 3de5a5b49580 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Wed Oct 03 01:19:19 2012 +0300 +++ b/src/lib-storage/mail-storage.c Wed Oct 03 01:20:22 2012 +0300 @@ -1680,11 +1680,19 @@ { struct mail_save_context *ctx = *_ctx; struct mail_keywords *keywords = ctx->keywords; + struct mail_private *mail; *_ctx = NULL; ctx->transaction->box->v.save_cancel(ctx); if (keywords != NULL) mailbox_keywords_unref(&keywords); + if (ctx->dest_mail != NULL) { + /* the dest_mail is no longer valid. if we're still saving + more mails, the mail sequence may get reused. make sure + the mail gets reset in between */ + mail = (struct mail_private *)ctx->dest_mail; + mail->v.close(&mail->mail); + } } struct mailbox_transaction_context * From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: lib-storage: struct mail.close() now clears all of ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c3dac1a94cf changeset: 15293:4c3dac1a94cf user: Timo Sirainen date: Wed Oct 03 01:19:19 2012 +0300 description: lib-storage: struct mail.close() now clears all of its data. diffstat: src/lib-storage/index/index-mail.c | 51 +++++++++++++++++++------------------ src/lib-storage/index/index-mail.h | 2 + 2 files changed, 28 insertions(+), 25 deletions(-) diffs (104 lines): diff -r 8c06fe0b280d -r 4c3dac1a94cf src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Oct 03 01:12:13 2012 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Oct 03 01:19:19 2012 +0300 @@ -1180,32 +1180,10 @@ index_mail_close_streams_full(mail, FALSE); } -void index_mail_close(struct mail *_mail) -{ - struct index_mail *mail = (struct index_mail *)_mail; - - /* If uid == 0 but seq != 0, we came here from saving a (non-mbox) - message. If that happens, don't bother checking if anything should - be cached since it was already checked. Also by now the transaction - may have already been rollbacked and seq point to a nonexistent - message. */ - if (mail->mail.mail.uid != 0) { - index_mail_cache_sizes(mail); - index_mail_cache_dates(mail); - } - - index_mail_close_streams_full(mail, TRUE); - - if (mail->data.wanted_headers != NULL) - mailbox_header_lookup_unref(&mail->data.wanted_headers); -} - -static void index_mail_reset(struct index_mail *mail) +static void index_mail_reset_data(struct index_mail *mail) { struct index_mail_data *data = &mail->data; - mail->mail.v.close(&mail->mail.mail); - memset(data, 0, sizeof(*data)); p_clear(mail->data_pool); @@ -1230,6 +1208,28 @@ mail->mail.mail.saving = FALSE; } +void index_mail_close(struct mail *_mail) +{ + struct index_mail *mail = (struct index_mail *)_mail; + + /* If uid == 0 but seq != 0, we came here from saving a (non-mbox) + message. If that happens, don't bother checking if anything should + be cached since it was already checked. Also by now the transaction + may have already been rollbacked and seq point to a nonexistent + message. */ + if (mail->mail.mail.uid != 0) { + index_mail_cache_sizes(mail); + index_mail_cache_dates(mail); + } + + index_mail_close_streams_full(mail, TRUE); + + if (mail->data.wanted_headers != NULL) + mailbox_header_lookup_unref(&mail->data.wanted_headers); + if (!mail->freeing) + index_mail_reset_data(mail); +} + static void check_envelope(struct index_mail *mail) { struct mail *_mail = &mail->mail.mail; @@ -1390,7 +1390,7 @@ if (mail->data.seq == seq) return; - index_mail_reset(mail); + mail->mail.v.close(&mail->mail.mail); mail->data.seq = seq; mail->mail.mail.seq = seq; @@ -1458,7 +1458,7 @@ index_mail_set_seq(_mail, seq, FALSE); return TRUE; } else { - index_mail_reset(mail); + mail->mail.v.close(&mail->mail.mail); mail->mail.mail.uid = uid; mail_set_expunged(&mail->mail.mail); return FALSE; @@ -1512,6 +1512,7 @@ directly */ i_assert(!mail->search_mail); + mail->freeing = TRUE; mail->mail.v.close(_mail); i_assert(_mail->transaction->mail_ref_count > 0); diff -r 8c06fe0b280d -r 4c3dac1a94cf src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Wed Oct 03 01:12:13 2012 +0300 +++ b/src/lib-storage/index/index-mail.h Wed Oct 03 01:19:19 2012 +0300 @@ -146,6 +146,8 @@ unsigned int pop3_state_set:1; /* mail created by mailbox_search_*() */ unsigned int search_mail:1; + /* close() is being called from mail_free() */ + unsigned int freeing:1; }; struct mail * From dovecot at dovecot.org Mon Oct 29 17:55:53 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:53 +0200 Subject: dovecot-2.2: mdbox: Fix to handling transactions with partially ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/19732dd98602 changeset: 15295:19732dd98602 user: Timo Sirainen date: Wed Oct 03 01:26:42 2012 +0300 description: mdbox: Fix to handling transactions with partially failed saves. diffstat: src/lib-storage/index/dbox-common/dbox-file.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (28 lines): diff -r 3de5a5b49580 -r 19732dd98602 src/lib-storage/index/dbox-common/dbox-file.c --- a/src/lib-storage/index/dbox-common/dbox-file.c Wed Oct 03 01:20:22 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Wed Oct 03 01:26:42 2012 +0300 @@ -532,7 +532,8 @@ { struct mail_storage *storage = &ctx->file->storage->storage; - if (ctx->last_flush_offset == ctx->output->offset) + if (ctx->last_flush_offset == ctx->output->offset && + ctx->last_checkpoint_offset == ctx->output->offset) return 0; if (o_stream_flush(ctx->output) < 0) { @@ -540,6 +541,14 @@ return -1; } + if (ctx->last_checkpoint_offset != ctx->output->offset) { + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + o_stream_seek(ctx->output, ctx->last_checkpoint_offset); + } + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { if (fdatasync(ctx->file->fd) < 0) { dbox_file_set_syscall_error(ctx->file, "fdatasync()"); From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: sdbox: Fix to handling transactions with partially ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b99bead8b339 changeset: 15296:b99bead8b339 user: Timo Sirainen date: Wed Oct 03 01:38:20 2012 +0300 description: sdbox: Fix to handling transactions with partially failed saves. diffstat: src/lib-storage/index/dbox-single/sdbox-save.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (31 lines): diff -r 19732dd98602 -r b99bead8b339 src/lib-storage/index/dbox-single/sdbox-save.c --- a/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:26:42 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Wed Oct 03 01:38:20 2012 +0300 @@ -168,7 +168,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) { struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; - struct dbox_file *const *files; + struct dbox_file **files; ctx->ctx.finished = TRUE; if (ctx->ctx.dbox_output == NULL) @@ -192,14 +192,17 @@ if (ctx->ctx.failed) { mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq); dbox_file_append_rollback(&ctx->append_ctx); + dbox_file_unlink(*files); + dbox_file_unref(files); + array_delete(&ctx->files, array_count(&ctx->files) - 1, 1); } else { dbox_file_append_checkpoint(ctx->append_ctx); if (dbox_file_append_commit(&ctx->append_ctx) < 0) ctx->ctx.failed = TRUE; + dbox_file_close(*files); } i_stream_unref(&ctx->ctx.input); - dbox_file_close(*files); ctx->ctx.dbox_output = NULL; return ctx->ctx.failed ? -1 : 0; From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: imap: LIST (SPECIAL-USE) shouldn't send INBOX reply. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12606087a774 changeset: 15297:12606087a774 user: Timo Sirainen date: Wed Oct 03 05:20:29 2012 +0300 description: imap: LIST (SPECIAL-USE) shouldn't send INBOX reply. diffstat: src/imap/cmd-list.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r b99bead8b339 -r 12606087a774 src/imap/cmd-list.c --- a/src/imap/cmd-list.c Wed Oct 03 01:38:20 2012 +0300 +++ b/src/imap/cmd-list.c Wed Oct 03 05:20:29 2012 +0300 @@ -878,7 +878,8 @@ /* INBOX always exists */ if (!ctx->inbox_found && ctx->cur_ns_match_inbox && (ctx->ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && - (ctx->list_flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0) { + (ctx->list_flags & (MAILBOX_LIST_ITER_SELECT_SUBSCRIBED | + MAILBOX_LIST_ITER_SELECT_SPECIALUSE)) == 0) { str = t_str_new(64); str_append(str, "* LIST (\\Unmarked) "); list_reply_append_ns_sep_param(str, From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Fix for handling view syncing for alread... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4308d1794328 changeset: 15298:4308d1794328 user: Timo Sirainen date: Wed Oct 03 05:41:46 2012 +0300 description: lib-index: Fix for handling view syncing for already deleted transaction logs. The sync changes' hidden-flag was set randomly, which could have caused flag changes to get lost. diffstat: src/lib-index/mail-index-view-sync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 12606087a774 -r 4308d1794328 src/lib-index/mail-index-view-sync.c --- a/src/lib-index/mail-index-view-sync.c Wed Oct 03 05:20:29 2012 +0300 +++ b/src/lib-index/mail-index-view-sync.c Wed Oct 03 05:41:46 2012 +0300 @@ -767,7 +767,7 @@ /* skip internal flag changes */ if (ctx->data_offset == ctx->hdr->size) - return 0; + return FALSE; update = CONST_PTR_OFFSET(data, ctx->data_offset); } @@ -835,6 +835,7 @@ sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS; sync_rec->uid1 = range[ctx->lost_flag_idx].seq1; sync_rec->uid2 = range[ctx->lost_flag_idx].seq2; + sync_rec->hidden = FALSE; ctx->lost_flag_idx++; return TRUE; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-storage: When configuring mailbox INBOX {}, mak... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/16f2f71c91b1 changeset: 15299:16f2f71c91b1 user: Timo Sirainen date: Wed Oct 03 05:42:55 2012 +0300 description: lib-storage: When configuring mailbox INBOX {}, make sure INBOX is uppercased. diffstat: src/lib-storage/mailbox-list-iter.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 4308d1794328 -r 16f2f71c91b1 src/lib-storage/mailbox-list-iter.c --- a/src/lib-storage/mailbox-list-iter.c Wed Oct 03 05:41:46 2012 +0300 +++ b/src/lib-storage/mailbox-list-iter.c Wed Oct 03 05:42:55 2012 +0300 @@ -137,6 +137,11 @@ autobox = array_append_space(&actx->boxes); autobox->name = set->name; autobox->set = set; + if (strcasecmp(autobox->name, "INBOX") == 0) { + /* make sure duplicate INBOX/Inbox/etc. + won't get created */ + autobox->name = "INBOX"; + } } } } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-storage: Fixed potential memory leak when mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/28be06dbd65e changeset: 15300:28be06dbd65e user: Timo Sirainen date: Wed Oct 03 05:43:27 2012 +0300 description: lib-storage: Fixed potential memory leak when mailbox_transaction_commit_get_changes() failed diffstat: src/lib-storage/mail-storage.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 16f2f71c91b1 -r 28be06dbd65e src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Wed Oct 03 05:42:55 2012 +0300 +++ b/src/lib-storage/mail-storage.c Wed Oct 03 05:43:27 2012 +0300 @@ -1485,7 +1485,6 @@ /* Store changes temporarily so that plugins overriding transaction_commit() can look at them. */ - changes.pool = NULL; ret = mailbox_transaction_commit_get_changes(t, &changes); if (changes.pool != NULL) pool_unref(&changes.pool); @@ -1500,11 +1499,14 @@ int ret; t->box->transaction_count--; + changes_r->pool = NULL; *_t = NULL; T_BEGIN { ret = t->box->v.transaction_commit(t, changes_r); } T_END; + if (ret < 0 && changes_r->pool != NULL) + pool_unref(&changes_r->pool); return ret; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-storage: Don't crash when searching multiple ke... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0306792cc843 changeset: 15301:0306792cc843 user: Timo Sirainen date: Fri Oct 05 00:15:01 2012 +0300 description: lib-storage: Don't crash when searching multiple keywords. Fixed by simply removing the keyword merging code. mail_search_args_simplify() is called before mail_search_args_init(), so the keywords are still NULL and merging can't be done. Alternative fix would have been to add string array to mail_search_arg.value containing the keywords, but all of this is a pretty unnecessary optimization. diffstat: src/lib-storage/mail-search.c | 63 ------------------------------------------- 1 files changed, 0 insertions(+), 63 deletions(-) diffs (88 lines): diff -r 28be06dbd65e -r 0306792cc843 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Wed Oct 03 05:43:27 2012 +0300 +++ b/src/lib-storage/mail-search.c Fri Oct 05 00:15:01 2012 +0300 @@ -586,48 +586,14 @@ return TRUE; } -static struct mail_keywords * -mail_search_keywords_merge(struct mailbox *box, - struct mail_keywords **_kw1, - struct mail_keywords **_kw2) -{ - struct mail_keywords *kw1 = *_kw1, *kw2 = *_kw2; - struct mail_keywords *new_kw; - - i_assert(kw1->index == kw2->index); - T_BEGIN { - ARRAY_TYPE(keyword_indexes) new_indexes; - unsigned int i, j; - - t_array_init(&new_indexes, kw1->count + kw2->count + 1); - array_append(&new_indexes, kw1->idx, kw1->count); - for (i = 0; i < kw2->count; i++) { - /* don't add duplicates */ - for (j = 0; j < kw1->count; j++) { - if (kw1->idx[j] == kw2->idx[i]) - break; - } - if (j == kw1->count) - array_append(&new_indexes, kw2->idx+i, 1); - } - new_kw = mailbox_keywords_create_from_indexes(box, - &new_indexes); - } T_END; - mailbox_keywords_unref(_kw1); - mailbox_keywords_unref(_kw2); - return new_kw; -} - static void mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { struct mail_search_arg *sub, *prev = NULL; struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - struct mail_search_arg *prev_kw_arg, *prev_not_kw_arg; prev_flags_arg = prev_not_flags_arg = NULL; - prev_kw_arg = prev_not_kw_arg = NULL; while (args != NULL) { if (args->match_not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -689,35 +655,6 @@ } } - /* merge all keywords arguments */ - if (args->type == SEARCH_KEYWORDS && - !args->match_not && parent_and) { - if (prev_kw_arg == NULL) - prev_kw_arg = args; - else { - prev_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } else if (args->type == SEARCH_KEYWORDS && - args->match_not && !parent_and) { - if (prev_not_kw_arg == NULL) - prev_not_kw_arg = args; - else { - prev_not_kw_arg->value.keywords = - mail_search_keywords_merge(box, - &prev_not_kw_arg->value.keywords, - &args->value.keywords); - prev->next = args->next; - args = args->next; - continue; - } - } - prev = args; args = args->next; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: fts-lucene: doveadm fts rescan crashed with mailbox... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d96313048f17 changeset: 15302:d96313048f17 user: Timo Sirainen date: Mon Oct 08 03:14:12 2012 +0300 description: fts-lucene: doveadm fts rescan crashed with mailbox_list_index=yes diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 0306792cc843 -r d96313048f17 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Fri Oct 05 00:15:01 2012 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon Oct 08 03:14:12 2012 +0300 @@ -753,7 +753,8 @@ while ((info = mailbox_list_iter_next(iter)) != NULL) { box = mailbox_alloc(index->list, info->name, (enum mailbox_flags)0); - if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + if (mailbox_open(box) == 0 && + mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) == 0 && (guids == NULL || hash_table_lookup(guids, metadata.guid) == NULL)) { From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: master: If service { protocol } is set and not incl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d268e810c15 changeset: 15303:4d268e810c15 user: Timo Sirainen date: Mon Oct 08 08:53:54 2012 +0300 description: master: If service { protocol } is set and not included in "protocols", ignore its settings diffstat: src/master/master-settings.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (28 lines): diff -r d96313048f17 -r 4d268e810c15 src/master/master-settings.c --- a/src/master/master-settings.c Mon Oct 08 03:14:12 2012 +0300 +++ b/src/master/master-settings.c Mon Oct 08 08:53:54 2012 +0300 @@ -496,6 +496,13 @@ for (i = 0; i < count; i++) { struct service_settings *service = services[i]; + if (*service->protocol != '\0' && + !str_array_find((const char **)set->protocols_split, + service->protocol)) { + /* protocol not enabled, ignore its settings */ + continue; + } + if (*service->executable == '\0') { *error_r = t_strdup_printf("service(%s): " "executable is empty", service->name); @@ -550,9 +557,7 @@ } #endif - if (*service->protocol != '\0' && - str_array_find((const char **)set->protocols_split, - service->protocol)) { + if (*service->protocol != '\0') { /* each imap/pop3/lmtp process can use up a connection, although if service_count=1 it's only temporary */ if (service->service_count != 1 || From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: sdbox: Class didn't have MAIL_STORAGE_CLASS_FLAG_FI... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7bc24fc1667e changeset: 15304:7bc24fc1667e user: Timo Sirainen date: Fri Oct 12 00:29:41 2012 +0300 description: sdbox: Class didn't have MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG set This currently only meant that mail_prefetch_count setting wasn't working. diffstat: src/lib-storage/index/dbox-single/sdbox-storage.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4d268e810c15 -r 7bc24fc1667e src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 08 08:53:54 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Fri Oct 12 00:29:41 2012 +0300 @@ -369,7 +369,7 @@ struct mail_storage sdbox_storage = { .name = SDBOX_STORAGE_NAME, - .class_flags = 0, + .class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG, .v = { NULL, From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: sdbox: Make sure mail_attachment_fs=sis-queue isn't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9542732069ff changeset: 15305:9542732069ff user: Timo Sirainen date: Fri Oct 12 00:30:23 2012 +0300 description: sdbox: Make sure mail_attachment_fs=sis-queue isn't attempted to be used. It could be fixed, but nobody seems to have used it so far.. diffstat: src/lib-storage/index/dbox-common/dbox-storage.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (18 lines): diff -r 7bc24fc1667e -r 9542732069ff src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Fri Oct 12 00:29:41 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Fri Oct 12 00:30:23 2012 +0300 @@ -99,6 +99,14 @@ } else { name = t_strdup_until(set->mail_attachment_fs, args++); } + if (strcmp(name, "sis-queue") == 0 && + (_storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) != 0) { + /* FIXME: the deduplication part doesn't work, because + sdbox renames the files.. */ + *error_r = "mail_attachment_fs: " + "sis-queue not currently supported by sdbox"; + return -1; + } dir = mail_user_home_expand(_storage->user, set->mail_attachment_dir); storage->attachment_dir = p_strdup(_storage->pool, dir); From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Log a warning if locking transaction log... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e44579c5b52b changeset: 15306:e44579c5b52b user: Timo Sirainen date: Fri Oct 12 08:31:15 2012 +0300 description: lib-index: Log a warning if locking transaction log takes longer than 30 secs. diffstat: src/lib-index/mail-transaction-log-private.h | 1 + src/lib-index/mail-transaction-log.c | 7 +++++++ 2 files changed, 8 insertions(+), 0 deletions(-) diffs (49 lines): diff -r 9542732069ff -r e44579c5b52b src/lib-index/mail-transaction-log-private.h --- a/src/lib-index/mail-transaction-log-private.h Fri Oct 12 00:30:23 2012 +0300 +++ b/src/lib-index/mail-transaction-log-private.h Fri Oct 12 08:31:15 2012 +0300 @@ -10,6 +10,7 @@ mails. */ #define MAIL_TRANSCATION_LOG_LOCK_TIMEOUT (3*60) #define MAIL_TRANSCATION_LOG_LOCK_CHANGE_TIMEOUT (3*60) +#define MAIL_TRANSACTION_LOG_LOCK_WARN_SECS 30 /* Rotate when log is older than ROTATE_TIME and larger than MIN_SIZE */ #define MAIL_TRANSACTION_LOG_ROTATE_MIN_SIZE (1024*32) diff -r 9542732069ff -r e44579c5b52b src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Fri Oct 12 00:30:23 2012 +0300 +++ b/src/lib-index/mail-transaction-log.c Fri Oct 12 08:31:15 2012 +0300 @@ -416,6 +416,7 @@ int mail_transaction_log_lock_head(struct mail_transaction_log *log) { struct mail_transaction_log_file *file; + time_t lock_wait_started, lock_secs = 0; int ret = 0; if (!log->log_2_unlink_checked) { @@ -437,6 +438,7 @@ can lock it and don't see another file, we can be sure no-one is creating a new log at the moment */ + lock_wait_started = time(NULL); for (;;) { file = log->head; if (mail_transaction_log_file_lock(file) < 0) @@ -451,6 +453,7 @@ if (ret == 0 && log->head == file) { /* success */ + lock_secs = file->lock_created - lock_wait_started; break; } @@ -462,6 +465,10 @@ /* try again */ } + if (lock_secs > MAIL_TRANSACTION_LOG_LOCK_WARN_SECS) { + i_warning("Locking transaction log file %s took %ld seconds", + log->head->filepath, (long)lock_secs); + } i_assert(ret < 0 || log->head != NULL); return ret; From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-storage: Fixed crash when attempting to remove ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a79e1a0040de changeset: 15307:a79e1a0040de user: Timo Sirainen date: Fri Oct 12 08:48:25 2012 +0300 description: lib-storage: Fixed crash when attempting to remove subscriptions for nonexistent shared users. diffstat: src/lib-storage/list/mailbox-list-subscriptions.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r e44579c5b52b -r a79e1a0040de src/lib-storage/list/mailbox-list-subscriptions.c --- a/src/lib-storage/list/mailbox-list-subscriptions.c Fri Oct 12 08:31:15 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-subscriptions.c Fri Oct 12 08:48:25 2012 +0300 @@ -110,6 +110,8 @@ char sep; int ret; + /* src_list is subscriptions=yes, dest_list is subscriptions=no + (or the same as src_list) */ i_assert((src_list->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) != 0); if (dest_list->subscriptions == NULL) { @@ -151,8 +153,8 @@ i_warning("Subscriptions file %s: " "Removing invalid entry: %s", path, name); - (void)subsfile_set_subscribed(dest_list, path, - mailbox_list_get_temp_prefix(dest_list), + (void)subsfile_set_subscribed(src_list, path, + mailbox_list_get_temp_prefix(src_list), name, FALSE); } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: configure: Make sure MYSQL_LIBS has -lmysqlclient e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c8d55ba25f39 changeset: 15308:c8d55ba25f39 user: Timo Sirainen date: Fri Oct 12 23:05:43 2012 +0300 description: configure: Make sure MYSQL_LIBS has -lmysqlclient even if it's not explicitly found. diffstat: configure.in | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a79e1a0040de -r c8d55ba25f39 configure.in --- a/configure.in Fri Oct 12 08:48:25 2012 +0300 +++ b/configure.in Fri Oct 12 23:05:43 2012 +0300 @@ -2175,6 +2175,7 @@ AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, mysql_config, NO) if test $MYSQL_CONFIG = NO; then # based on code from PHP + MYSQL_LIBS="-lmysqlclient -lz -lm" for i in /usr /usr/local /usr/local/mysql; do for j in include include/mysql ""; do if test -r "$i/$j/mysql.h"; then From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-master: Fixed -i parameter hand... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0262ede193e5 changeset: 15309:0262ede193e5 user: Timo Sirainen date: Tue Oct 16 03:08:21 2012 +0300 description: lib-master: Fixed -i parameter handling. It previously worked only if the default config socket wasn't usable. diffstat: src/lib-master/master-service.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r c8d55ba25f39 -r 0262ede193e5 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Fri Oct 12 23:05:43 2012 +0300 +++ b/src/lib-master/master-service.c Tue Oct 16 03:08:21 2012 +0300 @@ -359,6 +359,7 @@ if (!get_instance_config(arg, &path)) i_fatal("Unknown instance name: %s", arg); service->config_path = i_strdup(path); + service->config_path_is_default = FALSE; break; case 'k': service->keep_environment = TRUE; From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: dict quota: Fixed a potential crash if quota recalc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/41aac09497ee changeset: 15310:41aac09497ee user: Timo Sirainen date: Tue Oct 16 03:34:51 2012 +0300 description: dict quota: Fixed a potential crash if quota recalculation was triggered at deinit. diffstat: src/plugins/quota/quota-dict.c | 14 ++++++++++---- src/plugins/quota/quota-dirsize.c | 1 + src/plugins/quota/quota-fs.c | 3 ++- src/plugins/quota/quota-maildir.c | 1 + src/plugins/quota/quota-private.h | 2 +- src/plugins/quota/quota-storage.c | 20 ++++++++++++++++++++ 6 files changed, 35 insertions(+), 6 deletions(-) diffs (123 lines): diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-dict.c --- a/src/plugins/quota/quota-dict.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-dict.c Tue Oct 16 03:34:51 2012 +0300 @@ -86,10 +86,8 @@ { struct dict_quota_root *root = (struct dict_quota_root *)_root; - if (root->dict != NULL) { - (void)dict_wait(root->dict); + if (root->dict != NULL) dict_deinit(&root->dict); - } i_free(root); } @@ -208,6 +206,13 @@ return 0; } +static void dict_quota_flush(struct quota_root *_root) +{ + struct dict_quota_root *root = (struct dict_quota_root *)_root; + + (void)dict_wait(root->dict); +} + struct quota_backend quota_backend_dict = { "dict", @@ -221,6 +226,7 @@ dict_quota_root_get_resources, dict_quota_get_resource, dict_quota_update, - NULL + NULL, + dict_quota_flush } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-dirsize.c --- a/src/plugins/quota/quota-dirsize.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-dirsize.c Tue Oct 16 03:34:51 2012 +0300 @@ -219,6 +219,7 @@ dirsize_quota_root_get_resources, dirsize_quota_get_resource, dirsize_quota_update, + NULL, NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-fs.c --- a/src/plugins/quota/quota-fs.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-fs.c Tue Oct 16 03:34:51 2012 +0300 @@ -831,7 +831,8 @@ fs_quota_get_resource, fs_quota_update, - fs_quota_match_box + fs_quota_match_box, + NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-maildir.c --- a/src/plugins/quota/quota-maildir.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-maildir.c Tue Oct 16 03:34:51 2012 +0300 @@ -913,6 +913,7 @@ maildir_quota_root_get_resources, maildir_quota_get_resource, maildir_quota_update, + NULL, NULL } }; diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-private.h --- a/src/plugins/quota/quota-private.h Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-private.h Tue Oct 16 03:34:51 2012 +0300 @@ -67,7 +67,7 @@ int (*update)(struct quota_root *root, struct quota_transaction_context *ctx); bool (*match_box)(struct quota_root *root, struct mailbox *box); - + void (*flush)(struct quota_root *root); }; struct quota_backend { diff -r 0262ede193e5 -r 41aac09497ee src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Tue Oct 16 03:08:21 2012 +0300 +++ b/src/plugins/quota/quota-storage.c Tue Oct 16 03:34:51 2012 +0300 @@ -366,14 +366,34 @@ return ret; } +static void quota_roots_flush(struct quota *quota) +{ + struct quota_root *const *roots; + unsigned int i, count; + + roots = array_get("a->roots, &count); + for (i = 0; i < count; i++) { + if (roots[i]->backend.v.flush != NULL) + roots[i]->backend.v.flush(roots[i]); + } +} + static void quota_mailbox_close(struct mailbox *box) { struct quota_mailbox *qbox = QUOTA_CONTEXT(box); + struct quota_user *quser = QUOTA_USER_CONTEXT(box->storage->user); /* sync_notify() may be called outside sync_begin()..sync_deinit(). make sure we apply changes at close time at latest. */ quota_mailbox_sync_commit(qbox); + /* make sure quota backend flushes all data. this could also be done + somewhat later, but user.deinit() is too late, since the flushing + can trigger quota recalculation which isn't safe to do anymore + at user.deinit() when most of the loaded plugins have already been + deinitialized. */ + quota_roots_flush(quser->quota); + qbox->module_ctx.super.close(box); } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: mail_cache_map() API cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/022d0d21e56d changeset: 15311:022d0d21e56d user: Timo Sirainen date: Thu Oct 18 05:10:29 2012 +0300 description: lib-index: mail_cache_map() API cleanup diffstat: src/lib-index/mail-cache-compress.c | 3 +- src/lib-index/mail-cache-fields.c | 46 +++++++++++++++++---------------- src/lib-index/mail-cache-lookup.c | 10 ++++-- src/lib-index/mail-cache-private.h | 3 +- src/lib-index/mail-cache-transaction.c | 17 +++++++----- src/lib-index/mail-cache.c | 36 ++++++++++++++++++++----- 6 files changed, 72 insertions(+), 43 deletions(-) diffs (truncated from 337 to 300 lines): diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:10:29 2012 +0300 @@ -353,6 +353,7 @@ uint32_t file_seq, old_offset; ARRAY_TYPE(uint32_t) ext_offsets; const uint32_t *offsets; + const void *data; unsigned int i, count; int fd, ret; @@ -440,7 +441,7 @@ if (cache->file_cache != NULL) file_cache_set_fd(cache->file_cache, cache->fd); - if (mail_cache_map(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) return -1; if (mail_cache_header_fields_read(cache) < 0) return -1; diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-fields.c Thu Oct 18 05:10:29 2012 +0300 @@ -198,11 +198,14 @@ return list; } -static int mail_cache_header_fields_get_offset(struct mail_cache *cache, - uint32_t *offset_r, bool map) +static int +mail_cache_header_fields_get_offset(struct mail_cache *cache, + uint32_t *offset_r, + const struct mail_cache_header_fields **field_hdr_r) { const struct mail_cache_header_fields *field_hdr; struct mail_cache_header_fields tmp_field_hdr; + const void *data; uint32_t offset = 0, next_offset; unsigned int next_count = 0; bool invalidate = FALSE; @@ -210,6 +213,8 @@ if (MAIL_CACHE_IS_UNUSABLE(cache)) { *offset_r = 0; + if (field_hdr_r != NULL) + *field_hdr_r = NULL; return 0; } @@ -228,16 +233,16 @@ invalidate = TRUE; if (cache->mmap_base != NULL) { - if (mail_cache_map(cache, offset, - sizeof(*field_hdr)) < 0) - return -1; - if (offset >= cache->mmap_length) { + ret = mail_cache_map(cache, offset, sizeof(*field_hdr), + &data); + if (ret <= 0) { + if (ret < 0) + return -1; mail_cache_set_corrupted(cache, "header field next_offset points outside file"); return -1; } - - field_hdr = CONST_PTR_OFFSET(cache->data, offset); + field_hdr = data; } else { /* if we need to follow multiple offsets to get to the last one, it's faster to just pread() the file @@ -270,7 +275,7 @@ if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT) cache->need_compress_file_seq = cache->hdr->file_seq; - if (map) { + if (field_hdr_r != NULL) { if (cache->file_cache != NULL && invalidate) { /* if this isn't the first header in file and we hadn't read this before, we can't trust that the cached @@ -278,17 +283,23 @@ file_cache_invalidate(cache->file_cache, offset, field_hdr->size); } - if (mail_cache_map(cache, offset, field_hdr->size) < 0) + ret = mail_cache_map(cache, offset, field_hdr->size, &data); + if (ret < 0) return -1; + if (ret == 0) { + mail_cache_set_corrupted(cache, + "header field size outside file"); + return -1; + } + *field_hdr_r = data; } - *offset_r = offset; return 0; } int mail_cache_header_fields_read(struct mail_cache *cache) { - const struct mail_cache_header_fields *field_hdr = NULL; + const struct mail_cache_header_fields *field_hdr; struct mail_cache_field field; const uint32_t *last_used, *sizes; const uint8_t *types, *decisions; @@ -299,7 +310,7 @@ time_t max_drop_time; uint32_t offset, i; - if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0) + if (mail_cache_header_fields_get_offset(cache, &offset, &field_hdr) < 0) return -1; if (offset == 0) { @@ -307,13 +318,6 @@ return 0; } - field_hdr = CONST_PTR_OFFSET(cache->data, offset); - if (offset + field_hdr->size > cache->mmap_length) { - mail_cache_set_corrupted(cache, - "field header points outside file"); - return -1; - } - /* check the fixed size of the header. name[] has to be checked separately */ if (field_hdr->size < sizeof(*field_hdr) + @@ -322,9 +326,7 @@ return -1; } - field_hdr = CONST_PTR_OFFSET(cache->data, offset); new_fields_count = field_hdr->fields_count; - if (new_fields_count != 0) { cache->file_field_map = i_realloc(cache->file_field_map, diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:10:29 2012 +0300 @@ -14,6 +14,7 @@ const struct mail_cache_record **rec_r) { const struct mail_cache_record *rec; + const void *data; i_assert(offset != 0); @@ -24,14 +25,15 @@ } /* we don't know yet how large the record is, so just guess */ - if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH) < 0) + if (mail_cache_map(cache, offset, sizeof(*rec) + CACHE_PREFETCH, + &data) < 0) return -1; if (offset + sizeof(*rec) > cache->mmap_length) { mail_cache_set_corrupted(cache, "record points outside file"); return -1; } - rec = CACHE_RECORD(cache, offset); + rec = data; if (rec->size < sizeof(*rec)) { mail_cache_set_corrupted(cache, "invalid record size"); @@ -39,9 +41,9 @@ } if (rec->size > CACHE_PREFETCH) { /* larger than we guessed. map the rest of the record. */ - if (mail_cache_map(cache, offset, rec->size) < 0) + if (mail_cache_map(cache, offset, rec->size, &data) < 0) return -1; - rec = CACHE_RECORD(cache, offset); + rec = data; } if (rec->size > cache->mmap_length || diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:10:29 2012 +0300 @@ -256,7 +256,8 @@ int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx, struct mail_cache_iterate_field *field_r); -int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size); +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r); void mail_cache_file_close(struct mail_cache *cache); int mail_cache_reopen(struct mail_cache *cache); diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache-transaction.c Thu Oct 18 05:10:29 2012 +0300 @@ -1091,6 +1091,8 @@ uint32_t new_offset) { const struct mail_cache_record *rec; + const void *data; + int ret; i_assert(cache->locked); @@ -1107,15 +1109,16 @@ records at the same time. we'd rather not lose those additions, so force the linking order to be new_offset -> old_offset if it isn't already. */ - if (mail_cache_map(cache, new_offset, sizeof(*rec)) < 0) - return -1; - if (new_offset + sizeof(*rec) > cache->mmap_length) { - mail_cache_set_corrupted(cache, - "Cache record offset %u points outside file", - new_offset); + ret = mail_cache_map(cache, new_offset, sizeof(*rec), &data); + if (ret <= 0) { + if (ret == 0) { + mail_cache_set_corrupted(cache, + "Cache record offset %u points outside file", + new_offset); + } return -1; } - rec = CACHE_RECORD(cache, new_offset); + rec = data; if (rec->prev_offset == old_offset) { /* link is already correct */ return 0; diff -r 41aac09497ee -r 022d0d21e56d src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Oct 16 03:34:51 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:10:29 2012 +0300 @@ -145,6 +145,7 @@ { struct mail_index_view *view; const struct mail_index_ext *ext; + const void *data; i_assert(!cache->locked); @@ -167,7 +168,7 @@ mail_cache_init_file_cache(cache); - if (mail_cache_map(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) return -1; if (mail_cache_header_fields_read(cache) < 0) @@ -265,8 +266,10 @@ return TRUE; } -int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size) +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) { + const void *data; ssize_t ret; cache->remap_counter++; @@ -309,13 +312,22 @@ cache->hdr = &cache->hdr_ro_copy; if (offset == 0) mail_cache_update_need_compress(cache); - return 0; + + data = file_cache_get_map(cache->file_cache, + &cache->mmap_length); + if (offset > cache->mmap_length) { + *data_r = NULL; + return 0; + } + *data_r = CONST_PTR_OFFSET(data, offset); + return offset + size > cache->mmap_length ? 0 : 1; } if (offset < cache->mmap_length && size <= cache->mmap_length - offset) { /* already mapped */ - return 0; + *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); + return 1; } if (cache->mmap_base != NULL) { @@ -355,11 +367,18 @@ cache->hdr = cache->data; if (offset == 0) mail_cache_update_need_compress(cache); - return 0; + if (offset > cache->mmap_length) { + *data_r = NULL; + return 0; + } + *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); + return offset + size > cache->mmap_length ? 0 : 1; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Code cleanup: Removed mail_cache.data Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0d1de37ad9d8 changeset: 15312:0d1de37ad9d8 user: Timo Sirainen date: Thu Oct 18 05:16:54 2012 +0300 description: lib-index: Code cleanup: Removed mail_cache.data diffstat: src/lib-index/mail-cache-lookup.c | 10 +++++++--- src/lib-index/mail-cache-private.h | 8 ++------ src/lib-index/mail-cache.c | 30 +++++++++--------------------- 3 files changed, 18 insertions(+), 30 deletions(-) diffs (176 lines): diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Thu Oct 18 05:16:54 2012 +0300 @@ -272,6 +272,7 @@ field_r->field_idx = field_idx; field_r->data = CONST_PTR_OFFSET(ctx->rec, ctx->pos); field_r->size = data_size; + field_r->offset = ctx->offset + ctx->pos; /* each record begins from 32bit aligned position */ ctx->pos += (data_size + sizeof(uint32_t)-1) & ~(sizeof(uint32_t)-1); @@ -446,8 +447,7 @@ lines_count = i; hdr_data = t_new(struct header_lookup_data, 1); - hdr_data->offset = (const char *)&lines[lines_count+1] - - (const char *)ctx->view->cache->data; + hdr_data->offset = field->offset + (lines_count+1) * sizeof(uint32_t); hdr_data->data_size = data_size; for (i = 0; i < lines_count; i++) { @@ -473,6 +473,7 @@ struct mail_cache_iterate_field field; struct header_lookup_context ctx; struct header_lookup_line *lines; + const void *data; const unsigned char *p, *start, *end; uint8_t *field_state; unsigned int i, count, max_field = 0; @@ -537,7 +538,10 @@ /* then start filling dest buffer from the headers */ for (i = 0; i < count; i++) { - start = CONST_PTR_OFFSET(cache->data, lines[i].data->offset); + if (mail_cache_map(cache, lines[i].data->offset, + lines[i].data->data_size, &data) <= 0) + return -1; + start = data; end = start + lines[i].data->data_size; /* find the end of the (multiline) header */ diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300 @@ -39,10 +39,6 @@ #define MAIL_CACHE_LOCK_TIMEOUT 10 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 -#define CACHE_RECORD(cache, offset) \ - ((const struct mail_cache_record *) \ - ((const char *) (cache)->data + offset)) - #define MAIL_CACHE_IS_UNUSABLE(cache) \ ((cache)->hdr == NULL) @@ -135,7 +131,6 @@ dev_t st_dev; void *mmap_base; - const void *data; size_t mmap_length; struct file_cache *file_cache; /* mail_cache_map() increases this always. */ @@ -203,8 +198,9 @@ struct mail_cache_iterate_field { unsigned int field_idx; + unsigned int size; const void *data; - unsigned int size; + uoff_t offset; }; struct mail_cache_lookup_iterate_ctx { diff -r 022d0d21e56d -r 0d1de37ad9d8 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 05:10:29 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300 @@ -59,7 +59,6 @@ file_cache_set_fd(cache->file_cache, -1); cache->mmap_base = NULL; - cache->data = NULL; cache->hdr = NULL; cache->mmap_length = 0; cache->last_field_header_offset = 0; @@ -214,10 +213,9 @@ cache->need_compress_file_seq = hdr->file_seq; } -static bool mail_cache_verify_header(struct mail_cache *cache) +static bool mail_cache_verify_header(struct mail_cache *cache, + const struct mail_cache_header *hdr) { - const struct mail_cache_header *hdr = cache->data; - /* check that the header is still ok */ if (cache->mmap_length < sizeof(struct mail_cache_header)) { mail_cache_set_corrupted(cache, "File too small"); @@ -278,7 +276,6 @@ size = sizeof(struct mail_cache_header); if (cache->file_cache != NULL) { - cache->data = NULL; cache->hdr = NULL; ret = file_cache_read(cache->file_cache, offset, size); @@ -295,26 +292,24 @@ return -1; } - cache->data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); + data = file_cache_get_map(cache->file_cache, + &cache->mmap_length); if (offset == 0) { - if (!mail_cache_verify_header(cache)) { + if (!mail_cache_verify_header(cache, data)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? cache->hdr->file_seq : 0; return -1; } - memcpy(&cache->hdr_ro_copy, cache->data, + memcpy(&cache->hdr_ro_copy, data, sizeof(cache->hdr_ro_copy)); } cache->hdr = &cache->hdr_ro_copy; if (offset == 0) mail_cache_update_need_compress(cache); - data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); if (offset > cache->mmap_length) { *data_r = NULL; return 0; @@ -350,13 +345,11 @@ cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length); if (cache->mmap_base == MAP_FAILED) { cache->mmap_base = NULL; - cache->data = NULL; mail_cache_set_syscall_error(cache, "mmap()"); return -1; } - cache->data = cache->mmap_base; - if (!mail_cache_verify_header(cache)) { + if (!mail_cache_verify_header(cache, cache->mmap_base)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? @@ -364,7 +357,7 @@ return -1; } - cache->hdr = cache->data; + cache->hdr = cache->mmap_base; if (offset == 0) mail_cache_update_need_compress(cache); if (offset > cache->mmap_length) { @@ -698,13 +691,8 @@ return -1; } - if (cache->file_cache != NULL) { + if (cache->file_cache != NULL) file_cache_write(cache->file_cache, data, size, offset); - - /* data pointer may change if file cache was grown */ - cache->data = file_cache_get_map(cache->file_cache, - &cache->mmap_length); - } return 0; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/51f2be9aa8ad changeset: 15313:51f2be9aa8ad user: Timo Sirainen date: Thu Oct 18 05:22:36 2012 +0300 description: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file. diffstat: src/lib-index/mail-cache-compress.c | 4 + src/lib-index/mail-cache-private.h | 8 ++- src/lib-index/mail-cache.c | 123 +++++++++++++++++++++++------------ src/lib-index/mail-index.h | 5 +- 4 files changed, 96 insertions(+), 44 deletions(-) diffs (232 lines): diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:22:36 2012 +0300 @@ -461,6 +461,9 @@ if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) return 0; + /* compression isn't very efficient with small read()s */ + cache->map_with_read = FALSE; + if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates the dotlock file we need. */ @@ -495,5 +498,6 @@ bool mail_cache_need_compress(struct mail_cache *cache) { return cache->need_compress_file_seq != 0 && + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 && !cache->index->readonly; } diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:22:36 2012 +0300 @@ -130,9 +130,14 @@ ino_t st_ino; dev_t st_dev; + size_t mmap_length; + /* a) mmaping the whole file */ void *mmap_base; - size_t mmap_length; + /* b) using file cache */ struct file_cache *file_cache; + /* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */ + uoff_t read_offset; + buffer_t *read_buf; /* mail_cache_map() increases this always. */ unsigned int remap_counter; @@ -169,6 +174,7 @@ unsigned int hdr_modified:1; unsigned int field_header_write_pending:1; unsigned int compressing:1; + unsigned int map_with_read:1; }; struct mail_cache_loop_track { diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:22:36 2012 +0300 @@ -7,6 +7,7 @@ #include "nfs-workarounds.h" #include "file-cache.h" #include "mmap-util.h" +#include "read-full.h" #include "write-full.h" #include "mail-cache-private.h" @@ -264,17 +265,87 @@ return TRUE; } +static int +mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, + const void *data, bool copy_hdr) +{ + if (offset == 0) { + const struct mail_cache_header *hdr = data; + + if (!mail_cache_verify_header(cache, hdr)) { + cache->need_compress_file_seq = + !MAIL_CACHE_IS_UNUSABLE(cache) && + cache->hdr->file_seq != 0 ? + cache->hdr->file_seq : 0; + return -1; + } + if (!copy_hdr) + cache->hdr = data; + else { + memcpy(&cache->hdr_ro_copy, data, + sizeof(cache->hdr_ro_copy)); + cache->hdr = &cache->hdr_ro_copy; + } + mail_cache_update_need_compress(cache); + } + + if (offset + size > cache->mmap_length) + return 0; + return 1; +} + +static int +mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) +{ + void *data; + ssize_t ret; + + if (cache->read_buf == NULL) { + cache->read_buf = + buffer_create_dynamic(default_pool, size); + } else if (cache->read_offset <= offset && + cache->read_offset + cache->read_buf->used >= offset+size) { + /* already mapped */ + *data_r = CONST_PTR_OFFSET(cache->mmap_base, + offset - cache->read_offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + } else { + buffer_set_used_size(cache->read_buf, 0); + } + data = buffer_append_space_unsafe(cache->read_buf, size); + ret = pread(cache->fd, data, size, offset); + if (ret < 0) { + if (errno != ESTALE) + mail_cache_set_syscall_error(cache, "read()"); + + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + return -1; + } + buffer_set_used_size(cache->read_buf, ret); + + cache->read_offset = offset; + cache->mmap_length = offset + size; + + *data_r = data; + return mail_cache_map_finish(cache, offset, size, data, TRUE); +} + int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { const void *data; ssize_t ret; - cache->remap_counter++; - if (size == 0) size = sizeof(struct mail_cache_header); + cache->remap_counter++; + if (cache->map_with_read) + return mail_cache_map_with_read(cache, offset, size, data_r); + if (cache->file_cache != NULL) { cache->hdr = NULL; @@ -294,28 +365,9 @@ data = file_cache_get_map(cache->file_cache, &cache->mmap_length); - - if (offset == 0) { - if (!mail_cache_verify_header(cache, data)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - memcpy(&cache->hdr_ro_copy, data, - sizeof(cache->hdr_ro_copy)); - } - cache->hdr = &cache->hdr_ro_copy; - if (offset == 0) - mail_cache_update_need_compress(cache); - - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(data, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(data, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); } if (offset < cache->mmap_length && @@ -348,24 +400,9 @@ mail_cache_set_syscall_error(cache, "mmap()"); return -1; } - - if (!mail_cache_verify_header(cache, cache->mmap_base)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - - cache->hdr = cache->mmap_base; - if (offset == 0) - mail_cache_update_need_compress(cache); - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(cache->mmap_base, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, FALSE); } static int mail_cache_try_open(struct mail_cache *cache) @@ -437,6 +474,8 @@ if (!MAIL_INDEX_IS_IN_MEMORY(index) && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0) cache->file_cache = file_cache_new(-1); + cache->map_with_read = + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0; cache->ext_id = mail_index_ext_register(index, "cache", 0, diff -r 0d1de37ad9d8 -r 51f2be9aa8ad src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-index.h Thu Oct 18 05:22:36 2012 +0300 @@ -27,7 +27,10 @@ MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100, /* If we run out of disk space, fail modifications instead of moving indexes to memory. */ - MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200 + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200, + /* We're only going to save new messages to the index. + Avoid unnecessary reads. */ + MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400 }; enum mail_index_header_compat_flags { From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-storage: Open index with MAIL_INDEX_OPEN_FLAG_S... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7c6215b2b7c changeset: 15314:b7c6215b2b7c user: Timo Sirainen date: Thu Oct 18 05:23:27 2012 +0300 description: lib-storage: Open index with MAIL_INDEX_OPEN_FLAG_SAVEONLY if mailbox has MAILBOX_FLAG_SAVEONLY diffstat: src/lib-storage/index/index-storage.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 51f2be9aa8ad -r b7c6215b2b7c src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Thu Oct 18 05:22:36 2012 +0300 +++ b/src/lib-storage/index/index-storage.c Thu Oct 18 05:23:27 2012 +0300 @@ -304,6 +304,8 @@ ibox->list_index_sync_ext_id = (uint32_t)-1; ibox->index_flags = MAIL_INDEX_OPEN_FLAG_CREATE | mail_storage_settings_to_index_flags(box->storage->set); + if ((box->flags & MAILBOX_FLAG_SAVEONLY) != 0) + ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_SAVEONLY; ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; MODULE_CONTEXT_SET(box, index_storage_module, ibox); From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Handle better race condition there dovec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bcef97fb1202 changeset: 15315:bcef97fb1202 user: Timo Sirainen date: Thu Oct 18 05:55:30 2012 +0300 description: lib-index: Handle better race condition there dovecot.index.log and .log.2 are the same link. diffstat: src/lib-index/mail-transaction-log-file.c | 15 ++++++++++----- src/lib-index/mail-transaction-log-private.h | 3 +-- src/lib-index/mail-transaction-log.c | 6 +++--- 3 files changed, 14 insertions(+), 10 deletions(-) diffs (83 lines): diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:55:30 2012 +0300 @@ -756,6 +756,9 @@ second log file and we're going to overwrite this first one. */ } + /* NOTE: here's a race condition where both .log and .log.2 + point to the same file. our reading code should ignore that + though by comparing the inodes. */ } if (file_dotlock_replace(dotlock, @@ -809,8 +812,7 @@ return 0; } -int mail_transaction_log_file_open(struct mail_transaction_log_file *file, - bool check_existing) +int mail_transaction_log_file_open(struct mail_transaction_log_file *file) { struct mail_index *index = file->log->index; unsigned int i; @@ -834,10 +836,13 @@ ignore_estale = i < MAIL_INDEX_ESTALE_RETRY_COUNT; if (mail_transaction_log_file_stat(file, ignore_estale) < 0) ret = -1; - else if (check_existing && - mail_transaction_log_file_is_dupe(file)) + else if (mail_transaction_log_file_is_dupe(file)) { + /* probably our already opened .log file has been + renamed to .log.2 and we're trying to reopen it. + also possible that hit a race condition where .log + and .log.2 are linked. */ return 0; - else { + } else { ret = mail_transaction_log_file_read_hdr(file, ignore_estale); } diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log-private.h --- a/src/lib-index/mail-transaction-log-private.h Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log-private.h Thu Oct 18 05:55:30 2012 +0300 @@ -117,8 +117,7 @@ const char *path); void mail_transaction_log_file_free(struct mail_transaction_log_file **file); -int mail_transaction_log_file_open(struct mail_transaction_log_file *file, - bool check_existing); +int mail_transaction_log_file_open(struct mail_transaction_log_file *file); int mail_transaction_log_file_create(struct mail_transaction_log_file *file, bool reset); int mail_transaction_log_file_lock(struct mail_transaction_log_file *file); diff -r b7c6215b2b7c -r bcef97fb1202 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Thu Oct 18 05:23:27 2012 +0300 +++ b/src/lib-index/mail-transaction-log.c Thu Oct 18 05:55:30 2012 +0300 @@ -84,7 +84,7 @@ return 0; file = mail_transaction_log_file_alloc(log, log->filepath); - if ((ret = mail_transaction_log_file_open(file, FALSE)) <= 0) { + if ((ret = mail_transaction_log_file_open(file)) <= 0) { /* leave the file for _create() */ log->open_file = file; return ret; @@ -322,7 +322,7 @@ } file = mail_transaction_log_file_alloc(log, log->filepath); - if (mail_transaction_log_file_open(file, FALSE) <= 0) { + if (mail_transaction_log_file_open(file) <= 0) { mail_transaction_log_file_free(&file); return -1; } @@ -400,7 +400,7 @@ /* see if we have it in log.2 file */ file = mail_transaction_log_file_alloc(log, log->filepath2); - if ((ret = mail_transaction_log_file_open(file, TRUE)) <= 0) { + if ((ret = mail_transaction_log_file_open(file)) <= 0) { mail_transaction_log_file_free(&file); return ret; } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Fixed handling of finding a duplicate do... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be50d12be960 changeset: 15316:be50d12be960 user: Timo Sirainen date: Thu Oct 18 06:00:18 2012 +0300 description: lib-index: Fixed handling of finding a duplicate dovecot.index.log file_seq Previously we assumed that the already opened file was always the wrong one, but more common was that the newly opened file was .log.2 which should have been deleted. diffstat: src/lib-index/mail-transaction-log-file.c | 47 +++++++++++++++++++----------- 1 files changed, 30 insertions(+), 17 deletions(-) diffs (67 lines): diff -r bcef97fb1202 -r be50d12be960 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Oct 18 05:55:30 2012 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Thu Oct 18 06:00:18 2012 +0300 @@ -443,6 +443,32 @@ } static int +mail_transaction_log_file_fail_dupe(struct mail_transaction_log_file *file) +{ + int ret; + + /* mark the old file corrupted. we can't safely remove + it from the list however, so return failure. */ + file->hdr.indexid = 0; + if (strcmp(file->filepath, file->log->head->filepath) != 0) { + /* only mark .2 corrupted, just to make sure we don't lose any + changes from .log in case we're somehow wrong */ + mail_transaction_log_mark_corrupted(file); + ret = 0; + } else { + ret = -1; + } + if (!file->corrupted) { + file->corrupted = TRUE; + mail_index_set_error(file->log->index, + "Transaction log %s: " + "duplicate transaction log sequence (%u)", + file->filepath, file->hdr.file_seq); + } + return ret; +} + +static int mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file, bool ignore_estale) { @@ -528,26 +554,13 @@ corrupted. */ for (f = file->log->files; f != NULL; f = f->next) { if (f->hdr.file_seq == file->hdr.file_seq) { - /* mark the old file corrupted. we can't safely remove - it from the list however, so return failure. */ - f->hdr.indexid = 0; if (strcmp(f->filepath, f->log->head->filepath) != 0) { - /* only mark .2 corrupted, just to make sure - we don't lose any changes from .log in case - we're somehow wrong */ - mail_transaction_log_mark_corrupted(f); - ret = 0; + /* old "f" is the .log.2 */ + return mail_transaction_log_file_fail_dupe(f); } else { - ret = -1; + /* new "file" is probably the .log.2 */ + return mail_transaction_log_file_fail_dupe(file); } - if (!f->corrupted) { - f->corrupted = TRUE; - mail_index_set_error(f->log->index, - "Transaction log %s: " - "duplicate transaction log sequence (%u)", - f->filepath, f->hdr.file_seq); - } - return ret; } } From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: login_log_format_elements: Added %{real_rip} variable. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/92364817f4ba changeset: 15317:92364817f4ba user: Timo Sirainen date: Thu Oct 18 06:21:25 2012 +0300 description: login_log_format_elements: Added %{real_rip} variable. It differs from %r when Dovecot proxy sends an updated client IP address. Patch by Jack Bates. diffstat: src/login-common/client-common.c | 3 +++ src/login-common/client-common.h | 1 + 2 files changed, 4 insertions(+), 0 deletions(-) diffs (38 lines): diff -r be50d12be960 -r 92364817f4ba src/login-common/client-common.c --- a/src/login-common/client-common.c Thu Oct 18 06:00:18 2012 +0300 +++ b/src/login-common/client-common.c Thu Oct 18 06:21:25 2012 +0300 @@ -97,6 +97,7 @@ client->set = set; client->local_ip = *local_ip; client->ip = *remote_ip; + client->real_ip = *remote_ip; client->fd = fd; client->tls = ssl; client->trusted = client_is_trusted(client); @@ -429,6 +430,7 @@ { 'k', NULL, "ssl_security" }, { 'e', NULL, "mail_pid" }, { '\0', NULL, "session" }, + { '\0', NULL, "real_rip" }, { '\0', NULL, NULL } }; @@ -478,6 +480,7 @@ tab[13].value = client->mail_pid == 0 ? "" : dec2str(client->mail_pid); tab[14].value = client_get_session_id(client); + tab[15].value = net_ip2addr(&client->real_ip); return tab; } diff -r be50d12be960 -r 92364817f4ba src/login-common/client-common.h --- a/src/login-common/client-common.h Thu Oct 18 06:00:18 2012 +0300 +++ b/src/login-common/client-common.h Thu Oct 18 06:21:25 2012 +0300 @@ -88,6 +88,7 @@ struct ip_addr local_ip; struct ip_addr ip; + struct ip_addr real_ip; unsigned int local_port, remote_port; struct ssl_proxy *ssl_proxy; const struct login_settings *set; From dovecot at dovecot.org Mon Oct 29 17:55:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:54 +0200 Subject: dovecot-2.2: lib-index: Crashfix for MAIL_INDEX_OPEN_FLAG_SAVEON... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/99aec8d4d6c6 changeset: 15318:99aec8d4d6c6 user: Timo Sirainen date: Thu Oct 18 06:45:39 2012 +0300 description: lib-index: Crashfix for MAIL_INDEX_OPEN_FLAG_SAVEONLY change. diffstat: src/lib-index/mail-cache.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 92364817f4ba -r 99aec8d4d6c6 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Thu Oct 18 06:21:25 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 06:45:39 2012 +0300 @@ -307,7 +307,7 @@ } else if (cache->read_offset <= offset && cache->read_offset + cache->read_buf->used >= offset+size) { /* already mapped */ - *data_r = CONST_PTR_OFFSET(cache->mmap_base, + *data_r = CONST_PTR_OFFSET(cache->read_buf->data, offset - cache->read_offset); return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); } else { From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: redis dict: Log an error if we get disconnected une... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a97c62fdd4ad changeset: 15319:a97c62fdd4ad user: Timo Sirainen date: Sun Oct 21 07:13:44 2012 +0300 description: redis dict: Log an error if we get disconnected unexpectedly. diffstat: src/lib-dict/dict-redis.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 99aec8d4d6c6 -r a97c62fdd4ad src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Thu Oct 18 06:45:39 2012 +0300 +++ b/src/lib-dict/dict-redis.c Sun Oct 21 07:13:44 2012 +0300 @@ -241,6 +241,8 @@ case 0: return; case -1: + if (conn->dict->ioloop != NULL) + i_error("redis: Disconnected unexpectedly"); redis_conn_destroy(_conn); return; default: From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: Fixed previous broken change for handling... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4c337f38ed6 changeset: 15320:e4c337f38ed6 user: Timo Sirainen date: Mon Oct 22 15:17:39 2012 +0300 description: director: Fixed previous broken change for handling USER-WEAK commands. diffstat: src/director/director-connection.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r a97c62fdd4ad -r e4c337f38ed6 src/director/director-connection.c --- a/src/director/director-connection.c Sun Oct 21 07:13:44 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:17:39 2012 +0300 @@ -725,7 +725,9 @@ bool weak = TRUE; int ret; - if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) != 0) + /* note that unlike other commands we don't want to just ignore + duplicate commands */ + if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) return ret > 0; if (str_array_length(args) != 2 || From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: Don't handle pending requests from all ar... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd646623a1a8 changeset: 15321:cd646623a1a8 user: Timo Sirainen date: Mon Oct 22 15:20:57 2012 +0300 description: director: Don't handle pending requests from all around the code. I'm not sure if this actually fixes any bugs, but it definitely makes the state cleaner. diffstat: src/director/director.c | 15 ++++++++++++++- src/director/director.h | 1 + 2 files changed, 15 insertions(+), 1 deletions(-) diffs (45 lines): diff -r e4c337f38ed6 -r cd646623a1a8 src/director/director.c --- a/src/director/director.c Mon Oct 22 15:17:39 2012 +0300 +++ b/src/director/director.c Mon Oct 22 15:20:57 2012 +0300 @@ -787,9 +787,20 @@ user->username_hash)); } +static void director_state_callback_timeout(struct director *dir) +{ + timeout_remove(&dir->to_callback); + dir->state_change_callback(dir); +} + void director_set_state_changed(struct director *dir) { - dir->state_change_callback(dir); + /* we may get called to here from various places. use a timeout to + make sure the state callback is called with a clean state. */ + if (dir->to_callback == NULL) { + dir->to_callback = + timeout_add(0, director_state_callback_timeout, dir); + } } void director_update_send(struct director *dir, struct director_host *src, @@ -866,6 +877,8 @@ timeout_remove(&dir->to_sync); if (dir->to_remove_dirs != NULL) timeout_remove(&dir->to_remove_dirs); + if (dir->to_callback != NULL) + timeout_remove(&dir->to_callback); while (array_count(&dir->dir_hosts) > 0) { hostp = array_idx(&dir->dir_hosts, 0); host = *hostp; diff -r e4c337f38ed6 -r cd646623a1a8 src/director/director.h --- a/src/director/director.h Mon Oct 22 15:17:39 2012 +0300 +++ b/src/director/director.h Mon Oct 22 15:20:57 2012 +0300 @@ -44,6 +44,7 @@ ARRAY_DEFINE(connections, struct director_connection *); struct timeout *to_reconnect; struct timeout *to_sync; + struct timeout *to_callback; /* current mail hosts */ struct mail_host_list *mail_hosts; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: Log more clearly why a request timeouts. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/10fae591707c changeset: 15322:10fae591707c user: Timo Sirainen date: Mon Oct 22 15:23:25 2012 +0300 description: director: Log more clearly why a request timeouts. diffstat: src/director/director-request.c | 51 +++++++++++++++++++++++++++++++++------- 1 files changed, 42 insertions(+), 9 deletions(-) diffs (156 lines): diff -r cd646623a1a8 -r 10fae591707c src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:20:57 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:23:25 2012 +0300 @@ -12,24 +12,44 @@ #define DIRECTOR_REQUEST_TIMEOUT_SECS 30 #define RING_NOCONN_WARNING_DELAY_MSECS (2*1000) +enum director_request_delay_reason { + REQUEST_DELAY_NONE = 0, + REQUEST_DELAY_RINGNOTHANDSHAKED, + REQUEST_DELAY_RINGNOTSYNCED, + REQUEST_DELAY_NOHOSTS, + REQUEST_DELAY_WEAK, + REQUEST_DELAY_KILL +}; + +static const char *delay_reason_strings[] = { + "unknown", + "ring not handshaked", + "ring not synced", + "no hosts", + "weak user", + "kill waiting" +}; + struct director_request { struct director *dir; time_t create_time; unsigned int username_hash; + enum director_request_delay_reason delay_reason; director_request_callback *callback; void *context; }; static const char * -director_request_get_timeout_error(struct director_request *request) +director_request_get_timeout_error(struct director_request *request, + struct user *user, string_t *str) { - string_t *str = t_str_new(128); - struct user *user; unsigned int secs; - str_printfa(str, "Timeout - queued for %u secs (", + str_truncate(str, 0); + str_printfa(str, "Timeout because %s - queued for %u secs (", + delay_reason_strings[request->delay_reason], (unsigned int)(ioloop_time - request->create_time)); if (request->dir->ring_last_sync_time == 0) @@ -42,8 +62,6 @@ str_printfa(str, "Ring not synced for %u secs", secs); } - user = user_directory_lookup(request->dir->users, - request->username_hash); if (user != NULL) { if (user->weak) str_append(str, ", weak user"); @@ -57,7 +75,9 @@ static void director_request_timeout(struct director *dir) { struct director_request **requestp, *request; + struct user *user; const char *errormsg; + string_t *str = t_str_new(128); while (array_count(&dir->pending_requests) > 0) { requestp = array_idx_modifiable(&dir->pending_requests, 0); @@ -67,8 +87,12 @@ DIRECTOR_REQUEST_TIMEOUT_SECS > ioloop_time) break; + user = user_directory_lookup(request->dir->users, + request->username_hash); + errormsg = director_request_get_timeout_error(request, + user, str); + array_delete(&dir->pending_requests, 0, 1); - errormsg = director_request_get_timeout_error(request); T_BEGIN { request->callback(NULL, errormsg, request->context); } T_END; @@ -127,13 +151,16 @@ ring_noconn_warning, dir); } -static bool director_request_existing(struct director *dir, struct user *user) +static bool +director_request_existing(struct director_request *request, struct user *user) { + struct director *dir = request->dir; struct mail_host *host; if (user->kill_state != USER_KILL_STATE_NONE) { /* delay processing this user's connections until its existing connections have been killed */ + request->delay_reason = REQUEST_DELAY_KILL; return FALSE; } if (dir->right == NULL && dir->ring_synced) { @@ -147,6 +174,7 @@ if (user->weak) { /* wait for user to become non-weak */ + request->delay_reason = REQUEST_DELAY_WEAK; return FALSE; } if (!user_directory_user_is_near_expiring(dir->users, user)) @@ -157,6 +185,7 @@ host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash); if (!dir->ring_synced) { /* try again later once ring is synced */ + request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; return FALSE; } if (user->host == host) { @@ -200,6 +229,7 @@ } else { user->weak = TRUE; director_update_user_weak(dir, dir->self_host, NULL, user); + request->delay_reason = REQUEST_DELAY_WEAK; return FALSE; } } @@ -213,24 +243,27 @@ if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ ring_log_delayed_warning(dir); + request->delay_reason = REQUEST_DELAY_RINGNOTHANDSHAKED; return FALSE; } user = user_directory_lookup(dir->users, request->username_hash); if (user != NULL) { - if (!director_request_existing(dir, user)) + if (!director_request_existing(request, user)) return FALSE; user_directory_refresh(dir->users, user); } else { if (!dir->ring_synced) { /* delay adding new users until ring is again synced */ ring_log_delayed_warning(dir); + request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; return FALSE; } host = mail_host_get_by_hash(dir->mail_hosts, request->username_hash); if (host == NULL) { /* all hosts have been removed */ + request->delay_reason = REQUEST_DELAY_NOHOSTS; return FALSE; } user = user_directory_add(dir->users, request->username_hash, From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: If user's weak-flag appears to have gotte... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7352d48b4071 changeset: 15323:7352d48b4071 user: Timo Sirainen date: Mon Oct 22 15:29:27 2012 +0300 description: director: If user's weak-flag appears to have gotten stuck, unset it. diffstat: src/director/director-request.c | 7 +++++++ src/director/user-directory.c | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletions(-) diffs (44 lines): diff -r 10fae591707c -r 7352d48b4071 src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:23:25 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:29:27 2012 +0300 @@ -91,6 +91,13 @@ request->username_hash); errormsg = director_request_get_timeout_error(request, user, str); + if (user != NULL && + request->delay_reason == REQUEST_DELAY_WEAK) { + /* weakness appears to have gotten stuck. this is a + bug, but try to fix it for future requests by + removing the weakness. */ + user->weak = FALSE; + } array_delete(&dir->pending_requests, 0, 1); T_BEGIN { diff -r 10fae591707c -r 7352d48b4071 src/director/user-directory.c --- a/src/director/user-directory.c Mon Oct 22 15:23:25 2012 +0300 +++ b/src/director/user-directory.c Mon Oct 22 15:29:27 2012 +0300 @@ -65,7 +65,22 @@ { time_t expire_timestamp = user->timestamp + dir->timeout_secs; - return expire_timestamp >= ioloop_time; + if (expire_timestamp >= ioloop_time) + return TRUE; + + if (user->kill_state != USER_KILL_STATE_NONE) { + /* don't free this user until the kill is finished */ + return TRUE; + } + + if (user->weak) { + if (expire_timestamp + USER_NEAR_EXPIRING_MAX >= ioloop_time) + return TRUE; + + i_warning("User %u weakness appears to be stuck, removing it", + user->username_hash); + } + return FALSE; } static void user_directory_drop_expired(struct user_directory *dir) From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: Minor code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1dcf0090648e changeset: 15324:1dcf0090648e user: Timo Sirainen date: Mon Oct 22 15:30:01 2012 +0300 description: director: Minor code cleanup diffstat: src/director/director-connection.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 7352d48b4071 -r 1dcf0090648e src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 22 15:29:27 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:30:01 2012 +0300 @@ -728,7 +728,7 @@ /* note that unlike other commands we don't want to just ignore duplicate commands */ if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) - return ret > 0; + return FALSE; if (str_array_length(args) != 2 || str_to_uint(args[0], &username_hash) < 0 || From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: Don't remove user's weak flag from notify... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c16a0182533f changeset: 15325:c16a0182533f user: Timo Sirainen date: Mon Oct 22 15:32:04 2012 +0300 description: director: Don't remove user's weak flag from notify connection. If notify connection worked properly, the weak flag should never have been set in the first place. And if it's just suddenly removed, it won't finish the pending requests properly. diffstat: src/director/notify-connection.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 1dcf0090648e -r c16a0182533f src/director/notify-connection.c --- a/src/director/notify-connection.c Mon Oct 22 15:30:01 2012 +0300 +++ b/src/director/notify-connection.c Mon Oct 22 15:32:04 2012 +0300 @@ -33,7 +33,6 @@ i_warning("notify: User %s refreshed too late " "(%d secs)", line, diff); } - user->weak = FALSE; user_directory_refresh(conn->dir->users, user); director_update_user(conn->dir, conn->dir->self_host, user); From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: director: -D parameter now enables extensive debug ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/48af47f2eb9c changeset: 15326:48af47f2eb9c user: Timo Sirainen date: Mon Oct 22 15:35:59 2012 +0300 description: director: -D parameter now enables extensive debug logging. diffstat: src/director/director-connection.c | 50 +++++++++++++++++++++---------------- src/director/director-request.c | 17 ++++++++++++ src/director/director.c | 33 ++++++++++++++++-------- src/director/director.h | 5 +++- src/director/main.c | 2 +- 5 files changed, 72 insertions(+), 35 deletions(-) diffs (truncated from 305 to 300 lines): diff -r c16a0182533f -r 48af47f2eb9c src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director-connection.c Mon Oct 22 15:35:59 2012 +0300 @@ -454,12 +454,15 @@ *user_r = user_directory_add(dir->users, username_hash, host, timestamp); (*user_r)->weak = weak; + dir_debug("user refresh: %u added", username_hash); return TRUE; } if (user->weak) { if (!weak) { /* removing user's weakness */ + dir_debug("user refresh: %u weakness removed", + username_hash); unset_weak_user = TRUE; user->weak = FALSE; ret = TRUE; @@ -469,6 +472,7 @@ } else if (weak && !user_directory_user_is_recently_updated(dir->users, user)) { /* mark the user as weak */ + dir_debug("user refresh: %u set weak", username_hash); user->weak = TRUE; ret = TRUE; } else if (user->host != host) { @@ -516,6 +520,8 @@ user_directory_refresh(dir->users, user); ret = TRUE; } + dir_debug("user refresh: %u refreshed timeout to %ld", + username_hash, (long)user->timestamp); if (unset_weak_user) { /* user is no longer weak. handle pending requests for @@ -938,7 +944,7 @@ unsigned int handshake_secs = time(NULL) - conn->created; string_t *str; - if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || dir->debug) { + if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || director_debug) { str = t_str_new(128); str_printfa(str, "director(%s): Handshake took %u secs, " "bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T, @@ -1065,10 +1071,8 @@ /* duplicate SYNC (which was sent just in case the previous one got lost) */ } else { - if (dir->debug) { - i_debug("Ring is synced (%s sent seq=%u)", - conn->name, seq); - } + dir_debug("Ring is synced (%s sent seq=%u)", + conn->name, seq); director_set_ring_synced(dir); } } else if (dir->right != NULL) { @@ -1130,23 +1134,18 @@ director_host_cmp_to_self(host, dir->right->host, dir->self_host) <= 0) { /* the old connection is the correct one */ - if (dir->debug) { - i_debug("Ignoring CONNECT request to %s " - "(current right is %s)", - host->name, dir->right->name); - } + dir_debug("Ignoring CONNECT request to %s (current right is %s)", + host->name, dir->right->name); return TRUE; } - if (dir->debug) { - if (dir->right == NULL) { - i_debug("Received CONNECT request to %s, " - "initializing right", host->name); - } else { - i_debug("Received CONNECT request to %s, " - "replacing current right %s", - host->name, dir->right->name); - } + if (dir->right == NULL) { + dir_debug("Received CONNECT request to %s, " + "initializing right", host->name); + } else { + dir_debug("Received CONNECT request to %s, " + "replacing current right %s", + host->name, dir->right->name); } /* connect here */ @@ -1254,6 +1253,8 @@ const char *cmd, *const *args; bool ret; + dir_debug("input: %s: %s", conn->name, line); + args = t_strsplit_tab(line); cmd = args[0]; args++; if (cmd == NULL) { @@ -1535,9 +1536,9 @@ *_conn = NULL; - if (dir->debug && conn->host != NULL) { - i_debug("Disconnecting from %s: %s", - conn->host->name, remote_reason); + if (conn->host != NULL) { + dir_debug("Disconnecting from %s: %s", + conn->host->name, remote_reason); } if (*remote_reason != '\0' && conn->minor_version >= DIRECTOR_VERSION_QUIT) { @@ -1629,6 +1630,11 @@ if (conn->output->closed || !conn->connected) return; + if (director_debug) T_BEGIN { + const char *const *lines = t_strsplit(data, "\n"); + for (; lines[1] != NULL; lines++) + dir_debug("output: %s: %s", conn->name, *lines); + } T_END; ret = o_stream_send(conn->output, data, len); if (ret != (off_t)len) { if (ret < 0) diff -r c16a0182533f -r 48af47f2eb9c src/director/director-request.c --- a/src/director/director-request.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director-request.c Mon Oct 22 15:35:59 2012 +0300 @@ -168,6 +168,8 @@ /* delay processing this user's connections until its existing connections have been killed */ request->delay_reason = REQUEST_DELAY_KILL; + dir_debug("request: %u waiting for kill to finish", + user->username_hash); return FALSE; } if (dir->right == NULL && dir->ring_synced) { @@ -182,6 +184,8 @@ if (user->weak) { /* wait for user to become non-weak */ request->delay_reason = REQUEST_DELAY_WEAK; + dir_debug("request: %u waiting for weakness", + request->username_hash); return FALSE; } if (!user_directory_user_is_near_expiring(dir->users, user)) @@ -193,6 +197,8 @@ if (!dir->ring_synced) { /* try again later once ring is synced */ request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; + dir_debug("request: %u waiting for sync for making weak", + request->username_hash); return FALSE; } if (user->host == host) { @@ -237,6 +243,7 @@ user->weak = TRUE; director_update_user_weak(dir, dir->self_host, NULL, user); request->delay_reason = REQUEST_DELAY_WEAK; + dir_debug("request: %u set to weak", request->username_hash); return FALSE; } } @@ -249,6 +256,8 @@ if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ + dir_debug("request: %u waiting for handshake", + request->username_hash); ring_log_delayed_warning(dir); request->delay_reason = REQUEST_DELAY_RINGNOTHANDSHAKED; return FALSE; @@ -259,11 +268,15 @@ if (!director_request_existing(request, user)) return FALSE; user_directory_refresh(dir->users, user); + dir_debug("request: %u refreshed timeout to %u", + request->username_hash, user->timestamp); } else { if (!dir->ring_synced) { /* delay adding new users until ring is again synced */ ring_log_delayed_warning(dir); request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; + dir_debug("request: %u waiting for sync for adding", + request->username_hash); return FALSE; } host = mail_host_get_by_hash(dir->mail_hosts, @@ -271,10 +284,14 @@ if (host == NULL) { /* all hosts have been removed */ request->delay_reason = REQUEST_DELAY_NOHOSTS; + dir_debug("request: %u waiting for hosts", + request->username_hash); return FALSE; } user = user_directory_add(dir->users, request->username_hash, host, ioloop_time); + dir_debug("request: %u added timeout to %u", + request->username_hash, user->timestamp); } i_assert(!user->weak); diff -r c16a0182533f -r 48af47f2eb9c src/director/director.c --- a/src/director/director.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director.c Mon Oct 22 15:35:59 2012 +0300 @@ -22,6 +22,8 @@ #define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000 #define DIRECTOR_DELAYED_DIR_REMOVE_MSECS (1000*30) +bool director_debug; + static bool director_is_self_ip_set(struct director *dir) { struct ip_addr ip; @@ -106,10 +108,8 @@ if (director_has_outgoing_connection(dir, host)) return 0; - if (dir->debug) { - i_debug("Connecting to %s:%u", - net_ip2addr(&host->ip), host->port); - } + dir_debug("Connecting to %s:%u", + net_ip2addr(&host->ip), host->port); port = dir->test_port != 0 ? dir->test_port : host->port; fd = net_connect_ip(&host->ip, port, &dir->self_ip); if (fd == -1) { @@ -239,8 +239,7 @@ "continuing delayed requests"); dir->ring_handshake_warning_sent = FALSE; } - if (dir->debug) - i_debug("Director ring handshaked"); + dir_debug("Director ring handshaked"); dir->ring_handshaked = TRUE; director_set_ring_synced(dir); @@ -383,11 +382,9 @@ return; } - if (dir->debug) { - i_debug("Ring is desynced (seq=%u, sending SYNC to %s)", - dir->sync_seq, dir->right == NULL ? "(nowhere)" : - director_connection_get_name(dir->right)); - } + dir_debug("Ring is desynced (seq=%u, sending SYNC to %s)", + dir->sync_seq, dir->right == NULL ? "(nowhere)" : + director_connection_get_name(dir->right)); /* send PINGs to our connections more rapidly until we've synced again. if the connection has actually died, we don't need to wait (and @@ -889,3 +886,17 @@ array_free(&dir->connections); i_free(dir); } + +void dir_debug(const char *fmt, ...) +{ + va_list args; + + if (!director_debug) + return; + + va_start(args, fmt); + T_BEGIN { + i_debug("%s", t_strdup_vprintf(fmt, args)); + } T_END; + va_end(args); +} diff -r c16a0182533f -r 48af47f2eb9c src/director/director.h --- a/src/director/director.h Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/director.h Mon Oct 22 15:35:59 2012 +0300 @@ -80,9 +80,10 @@ unsigned int ring_synced:1; unsigned int sync_frozen:1; unsigned int sync_pending:1; - unsigned int debug:1; }; +extern bool director_debug; + /* Create a new director. If listen_ip specifies an actual IP, it's used with listen_port for finding ourself from the director_servers setting. listen_port is used regardless by director_host_add_from_string() for hosts @@ -147,4 +148,6 @@ int director_connect_host(struct director *dir, struct director_host *host); +void dir_debug(const char *fmt, ...) ATTR_FORMAT(1, 2); + #endif diff -r c16a0182533f -r 48af47f2eb9c src/director/main.c --- a/src/director/main.c Mon Oct 22 15:32:04 2012 +0300 +++ b/src/director/main.c Mon Oct 22 15:35:59 2012 +0300 @@ -215,7 +215,7 @@ main_preinit(); master_service_init_finish(master_service); director->test_port = test_port; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: decode2text.sh: Assume xmlunzip exists in the same ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/09ed39c15584 changeset: 15327:09ed39c15584 user: Timo Sirainen date: Mon Oct 22 18:35:56 2012 +0300 description: decode2text.sh: Assume xmlunzip exists in the same directory as this script. This avoids hardcoding /usr/local/libexec/dovecot/ path in it. diffstat: src/plugins/fts/decode2text.sh | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r 48af47f2eb9c -r 09ed39c15584 src/plugins/fts/decode2text.sh --- a/src/plugins/fts/decode2text.sh Mon Oct 22 15:35:59 2012 +0300 +++ b/src/plugins/fts/decode2text.sh Mon Oct 22 18:35:56 2012 +0300 @@ -17,6 +17,7 @@ # } # } +libexec_dir=`dirname $0` content_type=$1 # The second parameter is the format's filename extension, which is used when @@ -66,7 +67,7 @@ cd $tempdir || exit 1 unzip -q "$path" 2>/dev/null || exit 0 find . -name "$name" -print0 | xargs -0 cat | - /usr/local/libexec/dovecot/xml2text + $libexec_dir/xml2text } wait_timeout() { From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: auth: Log a nicer message if client timeouts authen... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49bb6cc43d03 changeset: 15328:49bb6cc43d03 user: Timo Sirainen date: Mon Oct 22 18:59:20 2012 +0300 description: auth: Log a nicer message if client timeouts authentication in the middle. diffstat: src/auth/auth-request-handler.c | 13 ++++++------- 1 files changed, 6 insertions(+), 7 deletions(-) diffs (29 lines): diff -r 09ed39c15584 -r 49bb6cc43d03 src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Mon Oct 22 18:35:56 2012 +0300 +++ b/src/auth/auth-request-handler.c Mon Oct 22 18:59:20 2012 +0300 @@ -401,19 +401,18 @@ static void auth_request_timeout(struct auth_request *request) { - const char *str; + unsigned int secs = (unsigned int)(time(NULL) - request->last_access); - str = t_strdup_printf("Request %u.%u timeouted after %u secs, state=%d", - request->handler->client_pid, request->id, - (unsigned int)(time(NULL) - request->last_access), - request->state); if (request->state != AUTH_REQUEST_STATE_MECH_CONTINUE) { /* client's fault */ auth_request_log_error(request, request->mech->mech_name, - "%s", str); + "Request %u.%u timed out after %u secs, state=%d", + request->handler->client_pid, request->id, + secs, request->state); } else if (request->set->verbose) { auth_request_log_info(request, request->mech->mech_name, - "%s", str); + "Request timed out waiting for client to continue authentication " + "(%u secs)", secs); } auth_request_handler_remove(request->handler, request); } From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-storage: Listing multiple mailbox patterns may ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f351303887c changeset: 15329:8f351303887c user: Timo Sirainen date: Fri Oct 26 11:17:51 2012 +0300 description: lib-storage: Listing multiple mailbox patterns may have returned duplicates with fs layout. diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 49bb6cc43d03 -r 8f351303887c src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Mon Oct 22 18:59:20 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:17:51 2012 +0300 @@ -447,7 +447,7 @@ /* sort the root dirs so that /foo is before /foo/bar */ array_sort(&ctx->roots, i_strcmp_p); /* remove /foo/bar when there already exists /foo parent */ - for (i = 1; i < array_count(&ctx->roots); i++) { + for (i = 1; i < array_count(&ctx->roots); ) { parentp = array_idx(&ctx->roots, i-1); childp = array_idx(&ctx->roots, i); parentlen = strlen(*parentp); @@ -456,6 +456,8 @@ (*childp)[parentlen] == ctx->sep || (*childp)[parentlen] == '\0')) array_delete(&ctx->roots, i, 1); + else + i++; } } From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: Increased initial memory pool size Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b41d2eec320b changeset: 15330:b41d2eec320b user: Timo Sirainen date: Fri Oct 26 11:33:22 2012 +0300 description: Increased initial memory pool size diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 8f351303887c -r b41d2eec320b src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:17:51 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 11:33:22 2012 +0300 @@ -490,7 +490,7 @@ flags); } - pool = pool_alloconly_create("mailbox list fs iter", 1024); + pool = pool_alloconly_create("mailbox list fs iter", 2048); ctx = p_new(pool, struct fs_list_iterate_context, 1); ctx->ctx.pool = pool; ctx->ctx.list = _list; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-index: If cache file unexpectedly shrinks in he... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7fde3830215e changeset: 15331:7fde3830215e user: Timo Sirainen date: Fri Oct 26 11:34:25 2012 +0300 description: lib-index: If cache file unexpectedly shrinks in header lookup, log an error. diffstat: src/lib-index/mail-cache-lookup.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (22 lines): diff -r b41d2eec320b -r 7fde3830215e src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Fri Oct 26 11:33:22 2012 +0300 +++ b/src/lib-index/mail-cache-lookup.c Fri Oct 26 11:34:25 2012 +0300 @@ -538,9 +538,16 @@ /* then start filling dest buffer from the headers */ for (i = 0; i < count; i++) { - if (mail_cache_map(cache, lines[i].data->offset, - lines[i].data->data_size, &data) <= 0) + ret = mail_cache_map(cache, lines[i].data->offset, + lines[i].data->data_size, &data); + if (ret <= 0) { + if (ret < 0) + return -1; + + mail_cache_set_corrupted(cache, + "header record unexpectedly points outside file"); return -1; + } start = data; end = start + lines[i].data->data_size; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-index: Optimize cache file reads with MAIL_INDE... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0efb98659a1f changeset: 15333:0efb98659a1f user: Timo Sirainen date: Fri Oct 26 11:43:05 2012 +0300 description: lib-index: Optimize cache file reads with MAIL_INDEX_OPEN_FLAG_SAVEONLY diffstat: src/lib-index/mail-cache-fields.c | 2 +- src/lib-index/mail-cache.c | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletions(-) diffs (38 lines): diff -r 3700fb6f8a42 -r 0efb98659a1f src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Fri Oct 26 11:37:07 2012 +0300 +++ b/src/lib-index/mail-cache-fields.c Fri Oct 26 11:43:05 2012 +0300 @@ -232,7 +232,7 @@ offset = next_offset; invalidate = TRUE; - if (cache->mmap_base != NULL) { + if (cache->mmap_base != NULL || cache->map_with_read) { ret = mail_cache_map(cache, offset, sizeof(*field_hdr), &data); if (ret <= 0) { diff -r 3700fb6f8a42 -r 0efb98659a1f src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Fri Oct 26 11:37:07 2012 +0300 +++ b/src/lib-index/mail-cache.c Fri Oct 26 11:43:05 2012 +0300 @@ -13,6 +13,8 @@ #include +#define MAIL_CACHE_MIN_HEADER_READ_SIZE 4096 + void mail_cache_set_syscall_error(struct mail_cache *cache, const char *function) { @@ -321,6 +323,13 @@ } else { buffer_set_used_size(cache->read_buf, 0); } + if (offset == 0 && size < MAIL_CACHE_MIN_HEADER_READ_SIZE) { + /* we can usually read the fields header after the cache + header. we need them both, so try to read them all with one + pread() call. */ + size = MAIL_CACHE_MIN_HEADER_READ_SIZE; + } + data = buffer_append_space_unsafe(cache->read_buf, size); ret = pread(cache->fd, data, size, offset); if (ret < 0) { From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: stats: Refresh user statistics during long maildir ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b1b693a69c5f changeset: 15334:b1b693a69c5f user: Timo Sirainen date: Fri Oct 26 12:09:03 2012 +0300 description: stats: Refresh user statistics during long maildir syncs. diffstat: src/plugins/stats/stats-plugin.c | 36 ++++++++++++++++++++++++++++++++++++ 1 files changed, 36 insertions(+), 0 deletions(-) diffs (60 lines): diff -r 0efb98659a1f -r b1b693a69c5f src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Fri Oct 26 11:43:05 2012 +0300 +++ b/src/plugins/stats/stats-plugin.c Fri Oct 26 12:09:03 2012 +0300 @@ -33,6 +33,13 @@ struct mailbox_transaction_stats prev_stats; }; +struct stats_storage { + union mail_storage_module_context module_ctx; + + struct mail_storage_callbacks old_callbacks; + void *old_context; +}; + struct stats_mailbox { union mailbox_module_context module_ctx; }; @@ -431,6 +438,33 @@ return ret; } +static void +stats_notify_ok(struct mailbox *box, const char *text, void *context) +{ + struct stats_storage *sstorage = STATS_CONTEXT(box->storage); + + /* most importantly we want to refresh stats for very long running + mailbox syncs */ + session_stats_refresh(box->storage->user); + + if (sstorage->old_callbacks.notify_ok != NULL) + sstorage->old_callbacks.notify_ok(box, text, context); +} + +static void stats_register_notify_callbacks(struct mail_storage *storage) +{ + struct stats_storage *sstorage = STATS_CONTEXT(storage); + + if (sstorage != NULL) + return; + + sstorage = p_new(storage->pool, struct stats_storage, 1); + sstorage->old_callbacks = storage->callbacks; + storage->callbacks.notify_ok = stats_notify_ok; + + MODULE_CONTEXT_SET(storage, stats_storage_module, sstorage); +} + static void stats_mailbox_allocated(struct mailbox *box) { struct mailbox_vfuncs *v = box->vlast; @@ -440,6 +474,8 @@ if (suser == NULL) return; + stats_register_notify_callbacks(box->storage); + sbox = p_new(box->pool, struct stats_mailbox, 1); sbox->module_ctx.super = *v; box->vlast = &sbox->module_ctx.super; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-storage: Fixed listing layout=fs when namespace... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22875bcaa952 changeset: 15335:22875bcaa952 user: Timo Sirainen date: Fri Oct 26 13:05:43 2012 +0300 description: lib-storage: Fixed listing layout=fs when namespace prefix part included wildcards. diffstat: src/lib-storage/list/mailbox-list-fs-iter.c | 23 +++++++++++++++++------ 1 files changed, 17 insertions(+), 6 deletions(-) diffs (34 lines): diff -r b1b693a69c5f -r 22875bcaa952 src/lib-storage/list/mailbox-list-fs-iter.c --- a/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 12:09:03 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Fri Oct 26 13:05:43 2012 +0300 @@ -395,13 +395,24 @@ for (patterns = ctx->valid_patterns; *patterns != NULL; patterns++) { pattern = *patterns; - for (p = last = pattern; *p != '\0'; p++) { - if (*p == '%' || *p == '*') - break; - if (*p == ns_sep) - last = p; + if (strncmp(pattern, ns->prefix, ns->prefix_len) != 0) { + /* typically e.g. prefix=foo/bar/, pattern=foo/%/% + we'll use root="" for this. + + it might of course also be pattern=foo/%/prefix/% + where we could optimize with root=prefix, but + probably too much trouble to implement. */ + prefix_vname = ""; + p = last = pattern; + } else { + for (p = last = pattern; *p != '\0'; p++) { + if (*p == '%' || *p == '*') + break; + if (*p == ns_sep) + last = p; + } + prefix_vname = t_strdup_until(pattern, last); } - prefix_vname = t_strdup_until(pattern, last); if (p == last+1 && *pattern == ns_sep) root = "/"; From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-index: After recent cache changes, cache was of... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3700fb6f8a42 changeset: 15332:3700fb6f8a42 user: Timo Sirainen date: Fri Oct 26 11:37:07 2012 +0300 description: lib-index: After recent cache changes, cache was often wrongly being thought of as unusable diffstat: src/lib-index/mail-cache.c | 28 +++++++++++++++++++--------- 1 files changed, 19 insertions(+), 9 deletions(-) diffs (88 lines): diff -r 7fde3830215e -r 3700fb6f8a42 src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Fri Oct 26 11:34:25 2012 +0300 +++ b/src/lib-index/mail-cache.c Fri Oct 26 11:37:07 2012 +0300 @@ -267,11 +267,13 @@ static int mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, - const void *data, bool copy_hdr) + const void *hdr_data, bool copy_hdr) { + const struct mail_cache_header *hdr = hdr_data; + if (offset == 0) { - const struct mail_cache_header *hdr = data; - + /* verify the header validity only with offset=0. this way + we won't waste time re-verifying it all the time */ if (!mail_cache_verify_header(cache, hdr)) { cache->need_compress_file_seq = !MAIL_CACHE_IS_UNUSABLE(cache) && @@ -279,14 +281,18 @@ cache->hdr->file_seq : 0; return -1; } + } + if (hdr_data != NULL) { if (!copy_hdr) - cache->hdr = data; + cache->hdr = hdr; else { - memcpy(&cache->hdr_ro_copy, data, + memcpy(&cache->hdr_ro_copy, hdr, sizeof(cache->hdr_ro_copy)); cache->hdr = &cache->hdr_ro_copy; } mail_cache_update_need_compress(cache); + } else { + i_assert(cache->hdr != NULL); } if (offset + size > cache->mmap_length) @@ -298,6 +304,7 @@ mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { + const void *hdr_data; void *data; ssize_t ret; @@ -309,7 +316,8 @@ /* already mapped */ *data_r = CONST_PTR_OFFSET(cache->read_buf->data, offset - cache->read_offset); - return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); } else { buffer_set_used_size(cache->read_buf, 0); } @@ -330,7 +338,8 @@ cache->mmap_length = offset + size; *data_r = data; - return mail_cache_map_finish(cache, offset, size, data, TRUE); + hdr_data = offset == 0 ? *data_r : NULL; + return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); } int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, @@ -367,7 +376,7 @@ &cache->mmap_length); *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(data, offset); - return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + return mail_cache_map_finish(cache, offset, size, data, TRUE); } if (offset < cache->mmap_length && @@ -402,7 +411,8 @@ } *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(cache->mmap_base, offset); - return mail_cache_map_finish(cache, offset, size, *data_r, FALSE); + return mail_cache_map_finish(cache, offset, size, + cache->mmap_base, FALSE); } static int mail_cache_try_open(struct mail_cache *cache) From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: lib-dict: Abort async transaction commits if client... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67e9cb0b06ec changeset: 15336:67e9cb0b06ec user: Timo Sirainen date: Mon Oct 29 16:36:59 2012 +0200 description: lib-dict: Abort async transaction commits if client gets disconnected from dict server. diffstat: src/lib-dict/dict-client.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (38 lines): diff -r 22875bcaa952 -r 67e9cb0b06ec src/lib-dict/dict-client.c --- a/src/lib-dict/dict-client.c Fri Oct 26 13:05:43 2012 +0300 +++ b/src/lib-dict/dict-client.c Mon Oct 29 16:36:59 2012 +0200 @@ -72,6 +72,7 @@ unsigned int failed:1; unsigned int sent_begin:1; + unsigned int async:1; }; static int client_dict_connect(struct client_dict *dict); @@ -444,9 +445,18 @@ static void client_dict_disconnect(struct client_dict *dict) { + struct client_dict_transaction_context *ctx, *next; + dict->connect_counter++; dict->handshaked = FALSE; + /* abort all pending async commits */ + for (ctx = dict->transactions; ctx != NULL; ctx = next) { + next = ctx->next; + if (ctx->async) + client_dict_finish_transaction(dict, ctx->id, -1); + } + if (dict->to_idle != NULL) timeout_remove(&dict->to_idle); if (dict->io != NULL) @@ -706,6 +716,7 @@ else if (async) { ctx->callback = callback; ctx->context = context; + ctx->async = TRUE; if (dict->async_commits++ == 0) { dict->io = io_add(dict->fd, IO_READ, dict_async_input, dict); From dovecot at dovecot.org Mon Oct 29 17:55:55 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 17:55:55 +0200 Subject: dovecot-2.2: Merged changes from v2.1 tree. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b46c1bf9d1e changeset: 15337:1b46c1bf9d1e user: Timo Sirainen date: Mon Oct 29 17:55:35 2012 +0200 description: Merged changes from v2.1 tree. diffstat: configure.in | 1 + src/auth/auth-request-handler.c | 13 +- src/director/director-connection.c | 56 ++-- src/director/director-request.c | 75 +++++- src/director/director.c | 48 +++- src/director/director.h | 6 +- src/director/main.c | 2 +- src/director/notify-connection.c | 1 - src/director/user-directory.c | 17 +- src/doveadm/doveadm-mail-altmove.c | 7 + src/doveadm/server-connection.c | 3 +- src/lib-dict/dict-client.c | 11 + src/lib-dict/dict-redis.c | 2 + src/lib-index/mail-cache-compress.c | 7 +- src/lib-index/mail-cache-fields.c | 48 ++-- src/lib-index/mail-cache-lookup.c | 27 +- src/lib-index/mail-cache-private.h | 19 +- src/lib-index/mail-cache-transaction.c | 17 +- src/lib-index/mail-cache.c | 168 ++++++++++---- src/lib-index/mail-index-view-sync.c | 3 +- src/lib-index/mail-index.h | 5 +- src/lib-index/mail-transaction-log-file.c | 62 +++- src/lib-index/mail-transaction-log-private.h | 4 +- src/lib-index/mail-transaction-log.c | 13 +- src/lib-master/master-service-settings-cache.c | 7 +- src/lib-master/master-service.c | 1 + src/lib-storage/index/dbox-common/dbox-file.c | 11 +- src/lib-storage/index/dbox-common/dbox-save.c | 16 +- src/lib-storage/index/dbox-common/dbox-storage.c | 9 + src/lib-storage/index/dbox-multi/mdbox-save.c | 2 - src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 +- src/lib-storage/index/dbox-single/sdbox-save.c | 13 +- src/lib-storage/index/dbox-single/sdbox-storage.c | 2 +- src/lib-storage/index/index-mail.c | 51 ++-- src/lib-storage/index/index-mail.h | 2 + src/lib-storage/index/index-storage.c | 2 + src/lib-storage/list/mailbox-list-fs-iter.c | 27 +- src/lib-storage/list/mailbox-list-iter.c | 5 + src/lib-storage/list/mailbox-list-subscriptions.c | 4 +- src/lib-storage/mail-search.c | 63 ----- src/lib-storage/mail-storage.c | 12 +- src/lib-storage/mailbox-list.c | 17 +- src/lib/compat.h | 4 + src/lib/ipwd.c | 4 + src/lmtp/lmtp-proxy.c | 1 + src/login-common/client-common.c | 3 + src/login-common/client-common.h | 1 + src/plugins/fts-lucene/lucene-wrapper.cc | 3 +- src/plugins/fts/decode2text.sh | 3 +- src/plugins/lazy-expunge/lazy-expunge-plugin.c | 8 +- src/plugins/quota/quota-dict.c | 14 +- src/plugins/quota/quota-dirsize.c | 1 + src/plugins/quota/quota-fs.c | 3 +- src/plugins/quota/quota-maildir.c | 1 + src/plugins/quota/quota-private.h | 2 +- src/plugins/quota/quota-storage.c | 20 + src/plugins/stats/stats-plugin.c | 36 +++ 57 files changed, 654 insertions(+), 315 deletions(-) diffs (truncated from 2376 to 300 lines): diff -r 0b4b2e37b793 -r 1b46c1bf9d1e configure.in --- a/configure.in Mon Oct 29 16:51:46 2012 +0200 +++ b/configure.in Mon Oct 29 17:55:35 2012 +0200 @@ -2178,6 +2178,7 @@ AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, mysql_config, NO) if test $MYSQL_CONFIG = NO; then # based on code from PHP + MYSQL_LIBS="-lmysqlclient -lz -lm" for i in /usr /usr/local /usr/local/mysql; do for j in include include/mysql ""; do if test -r "$i/$j/mysql.h"; then diff -r 0b4b2e37b793 -r 1b46c1bf9d1e src/auth/auth-request-handler.c --- a/src/auth/auth-request-handler.c Mon Oct 29 16:51:46 2012 +0200 +++ b/src/auth/auth-request-handler.c Mon Oct 29 17:55:35 2012 +0200 @@ -404,19 +404,18 @@ static void auth_request_timeout(struct auth_request *request) { - const char *str; + unsigned int secs = (unsigned int)(time(NULL) - request->last_access); - str = t_strdup_printf("Request %u.%u timeouted after %u secs, state=%d", - request->handler->client_pid, request->id, - (unsigned int)(time(NULL) - request->last_access), - request->state); if (request->state != AUTH_REQUEST_STATE_MECH_CONTINUE) { /* client's fault */ auth_request_log_error(request, request->mech->mech_name, - "%s", str); + "Request %u.%u timed out after %u secs, state=%d", + request->handler->client_pid, request->id, + secs, request->state); } else if (request->set->verbose) { auth_request_log_info(request, request->mech->mech_name, - "%s", str); + "Request timed out waiting for client to continue authentication " + "(%u secs)", secs); } auth_request_handler_remove(request->handler, request); } diff -r 0b4b2e37b793 -r 1b46c1bf9d1e src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 29 16:51:46 2012 +0200 +++ b/src/director/director-connection.c Mon Oct 29 17:55:35 2012 +0200 @@ -453,12 +453,15 @@ *user_r = user_directory_add(dir->users, username_hash, host, timestamp); (*user_r)->weak = weak; + dir_debug("user refresh: %u added", username_hash); return TRUE; } if (user->weak) { if (!weak) { /* removing user's weakness */ + dir_debug("user refresh: %u weakness removed", + username_hash); unset_weak_user = TRUE; user->weak = FALSE; ret = TRUE; @@ -468,6 +471,7 @@ } else if (weak && !user_directory_user_is_recently_updated(dir->users, user)) { /* mark the user as weak */ + dir_debug("user refresh: %u set weak", username_hash); user->weak = TRUE; ret = TRUE; } else if (user->host != host) { @@ -515,6 +519,8 @@ user_directory_refresh(dir->users, user); ret = TRUE; } + dir_debug("user refresh: %u refreshed timeout to %ld", + username_hash, (long)user->timestamp); if (unset_weak_user) { /* user is no longer weak. handle pending requests for @@ -725,8 +731,10 @@ bool weak = TRUE; int ret; - if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) != 0) - return ret > 0; + /* note that unlike other commands we don't want to just ignore + duplicate commands */ + if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) + return FALSE; if (str_array_length(args) != 2 || str_to_uint(args[0], &username_hash) < 0 || @@ -936,7 +944,7 @@ unsigned int handshake_secs = time(NULL) - conn->created; string_t *str; - if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || dir->debug) { + if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || director_debug) { str = t_str_new(128); str_printfa(str, "director(%s): Handshake took %u secs, " "bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T, @@ -1063,10 +1071,8 @@ /* duplicate SYNC (which was sent just in case the previous one got lost) */ } else { - if (dir->debug) { - i_debug("Ring is synced (%s sent seq=%u)", - conn->name, seq); - } + dir_debug("Ring is synced (%s sent seq=%u)", + conn->name, seq); director_set_ring_synced(dir); } } else if (dir->right != NULL) { @@ -1128,23 +1134,18 @@ director_host_cmp_to_self(host, dir->right->host, dir->self_host) <= 0) { /* the old connection is the correct one */ - if (dir->debug) { - i_debug("Ignoring CONNECT request to %s " - "(current right is %s)", - host->name, dir->right->name); - } + dir_debug("Ignoring CONNECT request to %s (current right is %s)", + host->name, dir->right->name); return TRUE; } - if (dir->debug) { - if (dir->right == NULL) { - i_debug("Received CONNECT request to %s, " - "initializing right", host->name); - } else { - i_debug("Received CONNECT request to %s, " - "replacing current right %s", - host->name, dir->right->name); - } + if (dir->right == NULL) { + dir_debug("Received CONNECT request to %s, " + "initializing right", host->name); + } else { + dir_debug("Received CONNECT request to %s, " + "replacing current right %s", + host->name, dir->right->name); } /* connect here */ @@ -1252,6 +1253,8 @@ const char *cmd, *const *args; bool ret; + dir_debug("input: %s: %s", conn->name, line); + args = t_strsplit_tab(line); cmd = args[0]; args++; if (cmd == NULL) { @@ -1531,9 +1534,9 @@ *_conn = NULL; - if (dir->debug && conn->host != NULL) { - i_debug("Disconnecting from %s: %s", - conn->host->name, remote_reason); + if (conn->host != NULL) { + dir_debug("Disconnecting from %s: %s", + conn->host->name, remote_reason); } if (*remote_reason != '\0' && conn->minor_version >= DIRECTOR_VERSION_QUIT) { @@ -1625,6 +1628,11 @@ if (conn->output->closed || !conn->connected) return; + if (director_debug) T_BEGIN { + const char *const *lines = t_strsplit(data, "\n"); + for (; lines[1] != NULL; lines++) + dir_debug("output: %s: %s", conn->name, *lines); + } T_END; ret = o_stream_send(conn->output, data, len); if (ret != (off_t)len) { if (ret < 0) diff -r 0b4b2e37b793 -r 1b46c1bf9d1e src/director/director-request.c --- a/src/director/director-request.c Mon Oct 29 16:51:46 2012 +0200 +++ b/src/director/director-request.c Mon Oct 29 17:55:35 2012 +0200 @@ -12,24 +12,44 @@ #define DIRECTOR_REQUEST_TIMEOUT_SECS 30 #define RING_NOCONN_WARNING_DELAY_MSECS (2*1000) +enum director_request_delay_reason { + REQUEST_DELAY_NONE = 0, + REQUEST_DELAY_RINGNOTHANDSHAKED, + REQUEST_DELAY_RINGNOTSYNCED, + REQUEST_DELAY_NOHOSTS, + REQUEST_DELAY_WEAK, + REQUEST_DELAY_KILL +}; + +static const char *delay_reason_strings[] = { + "unknown", + "ring not handshaked", + "ring not synced", + "no hosts", + "weak user", + "kill waiting" +}; + struct director_request { struct director *dir; time_t create_time; unsigned int username_hash; + enum director_request_delay_reason delay_reason; director_request_callback *callback; void *context; }; static const char * -director_request_get_timeout_error(struct director_request *request) +director_request_get_timeout_error(struct director_request *request, + struct user *user, string_t *str) { - string_t *str = t_str_new(128); - struct user *user; unsigned int secs; - str_printfa(str, "Timeout - queued for %u secs (", + str_truncate(str, 0); + str_printfa(str, "Timeout because %s - queued for %u secs (", + delay_reason_strings[request->delay_reason], (unsigned int)(ioloop_time - request->create_time)); if (request->dir->ring_last_sync_time == 0) @@ -42,8 +62,6 @@ str_printfa(str, "Ring not synced for %u secs", secs); } - user = user_directory_lookup(request->dir->users, - request->username_hash); if (user != NULL) { if (user->weak) str_append(str, ", weak user"); @@ -57,7 +75,9 @@ static void director_request_timeout(struct director *dir) { struct director_request **requestp, *request; + struct user *user; const char *errormsg; + string_t *str = t_str_new(128); while (array_count(&dir->pending_requests) > 0) { requestp = array_idx_modifiable(&dir->pending_requests, 0); @@ -67,8 +87,19 @@ DIRECTOR_REQUEST_TIMEOUT_SECS > ioloop_time) break; + user = user_directory_lookup(request->dir->users, + request->username_hash); + errormsg = director_request_get_timeout_error(request, + user, str); + if (user != NULL && + request->delay_reason == REQUEST_DELAY_WEAK) { + /* weakness appears to have gotten stuck. this is a + bug, but try to fix it for future requests by + removing the weakness. */ + user->weak = FALSE; + } + array_delete(&dir->pending_requests, 0, 1); - errormsg = director_request_get_timeout_error(request); T_BEGIN { request->callback(NULL, errormsg, request->context); } T_END; @@ -127,13 +158,18 @@ ring_noconn_warning, dir); } -static bool director_request_existing(struct director *dir, struct user *user) +static bool +director_request_existing(struct director_request *request, struct user *user) { + struct director *dir = request->dir; struct mail_host *host; if (user->kill_state != USER_KILL_STATE_NONE) { /* delay processing this user's connections until its existing connections have been killed */ + request->delay_reason = REQUEST_DELAY_KILL; + dir_debug("request: %u waiting for kill to finish", + user->username_hash); return FALSE; } if (dir->right == NULL && dir->ring_synced) { @@ -147,6 +183,9 @@ if (user->weak) { /* wait for user to become non-weak */ + request->delay_reason = REQUEST_DELAY_WEAK; + dir_debug("request: %u waiting for weakness", + request->username_hash); return FALSE; } if (!user_directory_user_is_near_expiring(dir->users, user)) From dovecot at dovecot.org Mon Oct 29 18:02:06 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 18:02:06 +0200 Subject: dovecot-2.2: Renamed configure.in to configure.ac. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/769be5b43fa6 changeset: 15338:769be5b43fa6 user: Timo Sirainen date: Mon Oct 29 18:01:54 2012 +0200 description: Renamed configure.in to configure.ac. Apparently automakes in future won't support configure.in anymore. diffstat: TODO | 2 + configure.ac | 2874 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ configure.in | 2874 ---------------------------------------------------------- 3 files changed, 2876 insertions(+), 2874 deletions(-) diffs (truncated from 5768 to 300 lines): diff -r 1b46c1bf9d1e -r 769be5b43fa6 TODO --- a/TODO Mon Oct 29 17:55:35 2012 +0200 +++ b/TODO Mon Oct 29 18:01:54 2012 +0200 @@ -10,6 +10,8 @@ imap-urlauth-client.c (AFTER destroying the worker) - special response in the control connection to make the imap-urlauth master wait before starting a new worker + - finish dsync rewrite + - imaptest: add condstore, qresync tests - settings parsing is horribly bloaty - doveadm: if running via doveadm-server and it fails, say something about diff -r 1b46c1bf9d1e -r 769be5b43fa6 configure.ac --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configure.ac Mon Oct 29 18:01:54 2012 +0200 @@ -0,0 +1,2874 @@ +AC_PREREQ([2.59]) +AC_INIT([Dovecot],[2.2.UNSTABLE],[dovecot at dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv0($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([foreign]) + +AM_MAINTAINER_MODE +PKG_PROG_PKG_CONFIG + +ACLOCAL_AMFLAGS='-I $(top_srcdir)' +AC_SUBST(ACLOCAL_AMFLAGS) + +dnl TEST_WITH(name, value, [plugin]) +AC_DEFUN([TEST_WITH], [ + want=want_`echo $1|sed s/-/_/g` + if test $2 = yes || test $2 = no || test $2 = auto; then + eval $want=$2 + elif test $2 = plugin; then + if test "$3" = plugin; then + eval $want=plugin + else + AC_ERROR([--with-$1=plugin not supported]) + fi + elif `echo $2|grep '^/' >/dev/null`; then + AC_ERROR([--with-$1=path not supported. You may want to use instead: +CPPFLAGS=-I$2/include LDFLAGS=-L$2/lib ./configure --with-$1]) + else + AC_ERROR([--with-$1: Unknown value: $2]) + fi +]) + +AC_ARG_ENABLE(devel-checks, +AS_HELP_STRING([--enable-devel-checks], [Enable some extra expensive checks for developers]), + if test x$enableval = xyes; then + AC_DEFINE(DEBUG,, Build with extra debugging checks) + want_devel_checks=yes + fi) + +AC_ARG_ENABLE(asserts, +AS_HELP_STRING([--enable-asserts], [Enable asserts (default)]), + if test x$enableval = xno; then + AC_DEFINE(DISABLE_ASSERTS,, Disable asserts) + fi) + +AC_ARG_WITH(shared-libs, +AS_HELP_STRING([--with-shared-libs], [Link binaries using shared Dovecot libraries (default)]), + want_shared_libs=$withval, + want_shared_libs=yes) +AM_CONDITIONAL(BUILD_SHARED_LIBS, test "$want_shared_libs" = "yes") + +AC_ARG_WITH(mem-align, +AS_HELP_STRING([--with-mem-align=BYTES], [Set the memory alignment (default: 8)]), + mem_align=$withval, + mem_align=8) + +AC_ARG_WITH(ioloop, +AS_HELP_STRING([--with-ioloop=IOLOOP], [Specify the I/O loop method to use (epoll, kqueue, poll; best for the fastest available; default is best)]), + ioloop=$withval, + ioloop=best) + +AC_ARG_WITH(notify, +AS_HELP_STRING([--with-notify=NOTIFY], [Specify the file system notification method to use (inotify, kqueue, dnotify, none; default is detected in the above order)]), + notify=$withval, + notify=) + +AC_ARG_WITH(nss, +AS_HELP_STRING([--with-nss], [Build with NSS module support (auto)]), + TEST_WITH(nss, $withval), + want_nss=auto) + +AC_ARG_WITH(shadow, +AS_HELP_STRING([--with-shadow], [Build with shadow password support (auto)]), + TEST_WITH(shadow, $withval), + want_shadow=auto) + +AC_ARG_WITH(pam, +AS_HELP_STRING([--with-pam], [Build with PAM support (auto)]), + TEST_WITH(pam, $withval), + want_pam=auto) + +AC_ARG_WITH(bsdauth, +AS_HELP_STRING([--with-bsdauth], [Build with BSD authentication support (auto)]), + TEST_WITH(bsdauth, $withval), + want_bsdauth=auto) + +AC_ARG_WITH(gssapi, +AS_HELP_STRING([--with-gssapi=yes|plugin Build with GSSAPI authentication support]), + TEST_WITH(gssapi, $withval, plugin), + want_gssapi=no) + +AC_ARG_WITH(sia, +AS_HELP_STRING([--with-sia], [Build with Tru64 SIA support]), + TEST_WITH(sia, $withval), + want_sia=no) + +AC_ARG_WITH(ldap, +AS_HELP_STRING([--with-ldap=yes|plugin], [Build with LDAP support]), + TEST_WITH(ldap, $withval, plugin), + want_ldap=no) + +AC_ARG_WITH(vpopmail, +AS_HELP_STRING([--with-vpopmail], [Build with vpopmail support (auto)]), + if test x$withval = xno; then + want_vpopmail=no + else + if test x$withval = xyes || test x$withval = xauto; then + vpopmail_home="`echo ~vpopmail`" + want_vpopmail=$withval + else + vpopmail_home="$withval" + want_vpopmail=yes + fi + fi, [ + want_vpopmail=no + ]) + +# Berkeley DB support is more or less broken. Disabled for now. +#AC_ARG_WITH(db, +#AS_HELP_STRING([--with-db], [Build with Berkeley DB support]), +# TEST_WITH(db, $withval), +# want_db=no) +want_db=no + +dnl The --with-sql is useful only if Dovecot is being built with all the SQL +dnl drivers as modules. If any SQL driver is built-in, this option is ignored. +AC_ARG_WITH(sql, +AS_HELP_STRING([--with-sql=yes|plugin], [Build with generic SQL support]), + TEST_WITH(sql, $withval, plugin), + want_sql=no) + +AC_ARG_WITH(pgsql, +AS_HELP_STRING([--with-pgsql], [Build with PostgreSQL driver support]), + TEST_WITH(pgsql, $withval), + want_pgsql=no) + +AC_ARG_WITH(mysql, +AS_HELP_STRING([--with-mysql], [Build with MySQL driver support]), + TEST_WITH(mysql, $withval), + want_mysql=no) + +AC_ARG_WITH(sqlite, +AS_HELP_STRING([--with-sqlite], [Build with SQLite3 driver support]), + TEST_WITH(sqlite, $withval), + want_sqlite=no) + +AC_ARG_WITH(lucene, +AS_HELP_STRING([--with-lucene], [Build with CLucene full text search support]), + TEST_WITH(lucene, $withval), + want_lucene=no) +AM_CONDITIONAL(BUILD_LUCENE, test "$want_lucene" = "yes") + +AC_ARG_WITH(stemmer, +AS_HELP_STRING([--with-stemmer], [Build with libstemmer support (for CLucene)]), + TEST_WITH(stemmer, $withval), + want_stemmer=auto) + +AC_ARG_WITH(solr, +AS_HELP_STRING([--with-solr], [Build with Solr full text search support]), + TEST_WITH(solr, $withval), + want_solr=no) + +AC_ARG_WITH(zlib, +AS_HELP_STRING([--with-zlib], [Build with zlib compression support]), + TEST_WITH(zlib, $withval), + want_zlib=auto) + +AC_ARG_WITH(bzlib, +AS_HELP_STRING([--with-bzlib], [Build with bzlib compression support]), + TEST_WITH(bzlib, $withval), + want_bzlib=auto) + +AC_ARG_WITH(libcap, +AS_HELP_STRING([--with-libcap], [Build with libcap support (Dropping capabilities).]), + TEST_WITH(libcap, $withval), + want_libcap=auto) + +AC_ARG_WITH(libwrap, +AS_HELP_STRING([--with-libwrap], [Build with libwrap, ie. TCP-wrappers]), + TEST_WITH(libwrap, $withval), + want_libwrap=no) + +AC_ARG_WITH(ssl, +AS_HELP_STRING([--with-ssl=gnutls|openssl], [Build with GNUTLS or OpenSSL (default)]), + if test x$withval = xno; then + want_gnutls=no + want_openssl=no + elif test x$withval = xgnutls; then + AC_ERROR([GNUTLS support is broken currently]) + want_gnutls=yes + want_openssl=no + elif test x$withval = xopenssl; then + want_gnutls=no + want_openssl=yes + elif test x$withval = xyes; then + want_gnutls=no + want_openssl=yes + else + AC_ERROR([--with-ssl: Invalid value: $withval]) + fi, [ + want_gnutls=no + want_openssl=auto + ]) + +AC_ARG_WITH(ssldir, +AS_HELP_STRING([--with-ssldir=DIR], [SSL base directory for certificates (/etc/ssl)]), + ssldir="$withval", + ssldir=/etc/ssl +) +AC_SUBST(ssldir) + +AC_ARG_WITH(rundir, +AS_HELP_STRING([--with-rundir=DIR], [Runtime data directory (LOCALSTATEDIR/run/dovecot)]), + rundir="$withval", + rundir=$localstatedir/run/$PACKAGE +) +AC_SUBST(rundir) + +AC_ARG_WITH(statedir, +AS_HELP_STRING([--with-statedir=DIR], [Permanent data directory (LOCALSTATEDIR/lib/dovecot)]), + statedir="$withval", + statedir=$localstatedir/lib/$PACKAGE +) +AC_SUBST(statedir) + +AC_ARG_WITH([systemdsystemunitdir], +AS_HELP_STRING([--with-systemdsystemunitdir=DIR], [Directory for systemd service files (auto=detect)]), [ + if test "$withval" = "auto"; then + systemdsystemunitdir=`$PKG_CONFIG --variable=systemdsystemunitdir systemd` + elif test "$withval" != "no"; then + systemdsystemunitdir=$withval + fi +], []) +if test "$systemdsystemunitdir" != ""; then + AC_SUBST(systemdsystemunitdir) + AC_DEFINE(HAVE_SYSTEMD,, Define if you want to use systemd socket activation) +fi +AM_CONDITIONAL(HAVE_SYSTEMD, test "$systemdsystemunitdir" != "") + +AC_ARG_WITH(gc, +AS_HELP_STRING([--with-gc], [Use Boehm garbage collector]), + TEST_WITH(gc, $withval), + want_gc=no) + +AC_ARG_WITH(storages, +AS_HELP_STRING([--with-storages], [Build with specified mail storage formats (mdbox sdbox maildir mbox cydir)]), [ + if test "$withval" = "yes" || test "$withval" = "no"; then + AC_MSG_ERROR([--with-storages needs storage list as parameter]) + fi + mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ], + mail_storages="shared mdbox sdbox maildir mbox cydir") +AC_SUBST(mail_storages) +mail_storages="$mail_storages imapc_stub pop3c_stub raw" +# drop duplicates +duplicates=`(for i in $mail_storages; do echo $i; done)|sort|uniq -d|xargs echo` +if test "$duplicates" != ""; then + AC_ERROR([Duplicate --with-storages: $duplicates]) +fi + +DC_DOVECOT_MODULEDIR + +AC_ARG_WITH(docs, +AS_HELP_STRING([--with-docs], [Install documentation (default)]), + if test x$withval = xno; then + want_docs=no + else + want_docs=yes + fi, + want_docs=yes) +AM_CONDITIONAL(BUILD_DOCS, test "$want_docs" = "yes") + +dnl always enable all of the passbs and userdbs that don't require extra libs +want_passwd=yes +want_passwd_file=yes +want_checkpassword=yes +want_prefetch_userdb=yes + +AC_ISC_POSIX +AC_PROG_CC +AC_PROG_CPP +AC_PROG_CXX # lucene plugin needs this +AC_HEADER_STDC +AC_C_INLINE +AC_PROG_LIBTOOL From dovecot at dovecot.org Mon Oct 29 18:29:15 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 18:29:15 +0200 Subject: dovecot-2.2: master: Removed unnecessary code duplication. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1fd0d04a8eb changeset: 15339:e1fd0d04a8eb user: Timo Sirainen date: Mon Oct 29 18:29:07 2012 +0200 description: master: Removed unnecessary code duplication. The settings parsing already does this. diffstat: src/master/service.c | 13 +------------ 1 files changed, 1 insertions(+), 12 deletions(-) diffs (37 lines): diff -r 769be5b43fa6 -r e1fd0d04a8eb src/master/service.c --- a/src/master/service.c Mon Oct 29 18:01:54 2012 +0200 +++ b/src/master/service.c Mon Oct 29 18:29:07 2012 +0200 @@ -231,6 +231,7 @@ service->idle_kill = set->idle_kill != 0 ? set->idle_kill : set->master_set->default_idle_kill; service->type = service->set->parsed_type; + service->executable = set->executable; if (set->process_limit == 0) { /* use default */ @@ -240,11 +241,6 @@ service->process_limit = set->process_limit; } - if (set->executable == NULL) { - *error_r = "executable not given"; - return NULL; - } - /* default gid to user's primary group */ if (get_uidgid(set->user, &service->uid, &service->gid, error_r) < 0) { switch (set->user_default) { @@ -290,13 +286,6 @@ } } - if (*set->executable == '/') - service->executable = set->executable; - else { - service->executable = - p_strconcat(pool, set->master_set->libexec_dir, "/", - set->executable, NULL); - } /* set these later, so if something fails we don't have to worry about closing them */ service->log_fd[0] = -1; From dovecot at dovecot.org Mon Oct 29 18:32:45 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 18:32:45 +0200 Subject: dovecot-2.2: master: Ignore service {} blocks with empty executa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f423090a363a changeset: 15340:f423090a363a user: Timo Sirainen date: Mon Oct 29 18:32:40 2012 +0200 description: master: Ignore service {} blocks with empty executable settings. diffstat: src/master/master-settings.c | 8 ++------ src/master/service.c | 9 ++++++++- 2 files changed, 10 insertions(+), 7 deletions(-) diffs (51 lines): diff -r e1fd0d04a8eb -r f423090a363a src/master/master-settings.c --- a/src/master/master-settings.c Mon Oct 29 18:29:07 2012 +0200 +++ b/src/master/master-settings.c Mon Oct 29 18:32:40 2012 +0200 @@ -505,12 +505,8 @@ continue; } - if (*service->executable == '\0') { - *error_r = t_strdup_printf("service(%s): " - "executable is empty", service->name); - return FALSE; - } - if (*service->executable != '/') { + if (*service->executable != '/' && + *service->executable != '\0') { service->executable = p_strconcat(pool, set->libexec_dir, "/", service->executable, NULL); diff -r e1fd0d04a8eb -r f423090a363a src/master/service.c --- a/src/master/service.c Mon Oct 29 18:29:07 2012 +0200 +++ b/src/master/service.c Mon Oct 29 18:32:40 2012 +0200 @@ -231,7 +231,6 @@ service->idle_kill = set->idle_kill != 0 ? set->idle_kill : set->master_set->default_idle_kill; service->type = service->set->parsed_type; - service->executable = set->executable; if (set->process_limit == 0) { /* use default */ @@ -357,6 +356,7 @@ return NULL; } + service->executable = set->executable; if (access(t_strcut(service->executable, ' '), X_OK) < 0) { *error_r = t_strdup_printf("access(%s) failed: %m", t_strcut(service->executable, ' ')); @@ -397,6 +397,13 @@ { char *const *proto; + if (*set->executable == '\0') { + /* silently allow service {} blocks for disabled extensions + (e.g. service managesieve {} block without pigeonhole + installed) */ + return FALSE; + } + if (*set->protocol == '\0') return TRUE; From dovecot at dovecot.org Mon Oct 29 19:04:52 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 19:04:52 +0200 Subject: dovecot-2.1: login proxy: For connect() failures log also the us... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/9d63f882194d changeset: 14791:9d63f882194d user: Timo Sirainen date: Mon Oct 29 19:04:36 2012 +0200 description: login proxy: For connect() failures log also the used local IP:port if available. diffstat: src/login-common/login-proxy.c | 41 +++++++++++++++++++++++++++++------------ 1 files changed, 29 insertions(+), 12 deletions(-) diffs (74 lines): diff -r 67e9cb0b06ec -r 9d63f882194d src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Mon Oct 29 16:36:59 2012 +0200 +++ b/src/login-common/login-proxy.c Mon Oct 29 19:04:36 2012 +0200 @@ -5,6 +5,7 @@ #include "istream.h" #include "ostream.h" #include "llist.h" +#include "str.h" #include "str-sanitize.h" #include "time-util.h" #include "master-service.h" @@ -194,16 +195,33 @@ proxy->state_rec = NULL; } +static void +proxy_log_connect_error(struct login_proxy *proxy) +{ + string_t *str = t_str_new(128); + struct ip_addr local_ip; + unsigned int local_port; + + str_printfa(str, "proxy(%s): connect(%s, %u) failed: %m (after %u secs", + proxy->client->virtual_user, + proxy->host, proxy->port, + (unsigned int)(ioloop_time - proxy->created.tv_sec)); + + if (proxy->server_fd != -1 && + net_getsockname(proxy->server_fd, &local_ip, &local_port) == 0) { + str_printfa(str, ", local=%s:%u", + net_ip2addr(&local_ip), local_port); + } + + str_append_c(str, ')'); + i_error("%s", str_c(str)); +} + static void proxy_wait_connect(struct login_proxy *proxy) { - int err; - - err = net_geterror(proxy->server_fd); - if (err != 0) { - i_error("proxy(%s): connect(%s, %u) failed: %s (after %u secs)", - proxy->client->virtual_user, - proxy->host, proxy->port, strerror(err), - (unsigned int)(ioloop_time - proxy->created.tv_sec)); + errno = net_geterror(proxy->server_fd); + if (errno != 0) { + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); return; @@ -229,8 +247,8 @@ static void proxy_connect_timeout(struct login_proxy *proxy) { - i_error("proxy(%s): connect(%s, %u) timed out", - proxy->client->virtual_user, proxy->host, proxy->port); + errno = ETIMEDOUT; + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); } @@ -252,8 +270,7 @@ proxy->server_fd = net_connect_ip(&proxy->ip, proxy->port, NULL); if (proxy->server_fd == -1) { - i_error("proxy(%s): connect(%s, %u) failed: %m", - proxy->client->virtual_user, proxy->host, proxy->port); + proxy_log_connect_error(proxy); login_proxy_free(&proxy); return -1; } From dovecot at dovecot.org Mon Oct 29 19:52:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 19:52:33 +0200 Subject: dovecot-2.2: Replaced *_INDEX_PREFIX macros with a common MAIL_I... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/15dfd6475f64 changeset: 15341:15dfd6475f64 user: Timo Sirainen date: Mon Oct 29 19:52:21 2012 +0200 description: Replaced *_INDEX_PREFIX macros with a common MAIL_INDEX_PREFIX. diffstat: src/lib-storage/index/cydir/cydir-storage.c | 3 +-- src/lib-storage/index/cydir/cydir-storage.h | 1 - src/lib-storage/index/dbox-common/dbox-storage.c | 2 +- src/lib-storage/index/dbox-common/dbox-storage.h | 1 - src/lib-storage/index/dbox-multi/mdbox-storage.c | 3 +-- src/lib-storage/index/dbox-single/sdbox-storage.c | 3 +-- src/lib-storage/index/imapc/imapc-storage.c | 3 +-- src/lib-storage/index/imapc/imapc-storage.h | 1 - src/lib-storage/index/maildir/maildir-storage.c | 3 +-- src/lib-storage/index/maildir/maildir-storage.h | 1 - src/lib-storage/index/mbox/mbox-storage.c | 3 +-- src/lib-storage/index/mbox/mbox-storage.h | 1 - src/lib-storage/index/pop3c/pop3c-storage.c | 3 +-- src/lib-storage/index/pop3c/pop3c-storage.h | 1 - src/lib-storage/mail-storage-private.h | 3 +++ src/plugins/virtual/virtual-storage.c | 3 +-- src/plugins/virtual/virtual-storage.h | 1 - 17 files changed, 12 insertions(+), 24 deletions(-) diffs (206 lines): diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/cydir/cydir-storage.c --- a/src/lib-storage/index/cydir/cydir-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/cydir/cydir-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -52,8 +52,7 @@ mbox->box.list = list; mbox->box.mail_vfuncs = &cydir_mail_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, flags, - CYDIR_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); mbox->storage = (struct cydir_storage *)storage; return &mbox->box; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/cydir/cydir-storage.h --- a/src/lib-storage/index/cydir/cydir-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/cydir/cydir-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -5,7 +5,6 @@ #define CYDIR_STORAGE_NAME "cydir" #define CYDIR_SUBSCRIPTION_FILE_NAME "subscriptions." -#define CYDIR_INDEX_PREFIX "dovecot.index" struct cydir_storage { struct mail_storage storage; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -153,7 +153,7 @@ if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir) <= 0) return; - path = t_strdup_printf("%s/"DBOX_INDEX_PREFIX".log", dir); + path = t_strdup_printf("%s/"MAIL_INDEX_PREFIX".log", dir); index_mailbox_check_add(box, path); } } diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/dbox-common/dbox-storage.h --- a/src/lib-storage/index/dbox-common/dbox-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -10,7 +10,6 @@ #define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" #define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" -#define DBOX_INDEX_PREFIX "dovecot.index" #define DBOX_TEMP_FILE_PREFIX ".temp." #define DBOX_ALT_SYMLINK_NAME "dbox-alt-root" diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -156,8 +156,7 @@ mbox->box.list = list; mbox->box.mail_vfuncs = &mdbox_mail_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, - flags, DBOX_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); ibox = INDEX_STORAGE_CONTEXT(&mbox->box); ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -105,8 +105,7 @@ mbox->box.list = list; mbox->box.mail_vfuncs = &sdbox_mail_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, - flags, DBOX_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); ibox = INDEX_STORAGE_CONTEXT(&mbox->box); ibox->index_flags |= MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS | diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -326,8 +326,7 @@ mbox->box.list = list; mbox->box.mail_vfuncs = &imapc_mail_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, flags, - IMAPC_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); mbox->storage = (struct imapc_storage *)storage; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -5,7 +5,6 @@ #include "imapc-settings.h" #define IMAPC_STORAGE_NAME "imapc" -#define IMAPC_INDEX_PREFIX "dovecot.index" #define IMAPC_LIST_ESCAPE_CHAR '%' #define IMAPC_LIST_BROKEN_CHAR '~' diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -281,8 +281,7 @@ mbox->box.mail_vfuncs = &maildir_mail_vfuncs; mbox->maildir_list_index_ext_id = (uint32_t)-1; - index_storage_mailbox_alloc(&mbox->box, vname, flags, - MAILDIR_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); mbox->storage = (struct maildir_storage *)storage; return &mbox->box; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/maildir/maildir-storage.h --- a/src/lib-storage/index/maildir/maildir-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -5,7 +5,6 @@ #define MAILDIR_STORAGE_NAME "maildir" #define MAILDIR_SUBSCRIPTION_FILE_NAME "subscriptions" -#define MAILDIR_INDEX_PREFIX "dovecot.index" #define MAILDIR_UIDVALIDITY_FNAME "dovecot-uidvalidity" /* "base,S=123:2," means: diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -366,8 +366,7 @@ mbox->box.list = list; mbox->box.mail_vfuncs = &mbox_mail_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, - flags, MBOX_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); mbox->storage = (struct mbox_storage *)storage; mbox->mbox_fd = -1; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/mbox/mbox-storage.h --- a/src/lib-storage/index/mbox/mbox-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -12,7 +12,6 @@ #define MBOX_STORAGE_NAME "mbox" #define MBOX_SUBSCRIPTION_FILE_NAME ".subscriptions" -#define MBOX_INDEX_PREFIX "dovecot.index" #define MBOX_INDEX_DIR_NAME ".imap" #define MBOX_UIDVALIDITY_FNAME "dovecot-uidvalidity" diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/pop3c/pop3c-storage.c --- a/src/lib-storage/index/pop3c/pop3c-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/pop3c/pop3c-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -114,8 +114,7 @@ mbox->box.mail_vfuncs = &pop3c_mail_vfuncs; mbox->storage = (struct pop3c_storage *)storage; - index_storage_mailbox_alloc(&mbox->box, vname, flags, - POP3C_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); return &mbox->box; } diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/index/pop3c/pop3c-storage.h --- a/src/lib-storage/index/pop3c/pop3c-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/index/pop3c/pop3c-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -4,7 +4,6 @@ #include "index-storage.h" #define POP3C_STORAGE_NAME "pop3c" -#define POP3C_INDEX_PREFIX "dovecot.index" struct pop3c_storage { struct mail_storage storage; diff -r f423090a363a -r 15dfd6475f64 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/lib-storage/mail-storage-private.h Mon Oct 29 19:52:21 2012 +0200 @@ -9,6 +9,9 @@ #include "mail-storage-settings.h" #include "mail-index-private.h" +/* Default prefix for indexes */ +#define MAIL_INDEX_PREFIX "dovecot.index" + /* Block size when read()ing message header. */ #define MAIL_READ_HDR_BLOCK_SIZE (1024*4) /* Block size when read()ing message (header and) body. */ diff -r f423090a363a -r 15dfd6475f64 src/plugins/virtual/virtual-storage.c --- a/src/plugins/virtual/virtual-storage.c Mon Oct 29 18:32:40 2012 +0200 +++ b/src/plugins/virtual/virtual-storage.c Mon Oct 29 19:52:21 2012 +0200 @@ -229,8 +229,7 @@ mbox->box.mail_vfuncs = &virtual_mail_vfuncs; mbox->vfuncs = virtual_mailbox_vfuncs; - index_storage_mailbox_alloc(&mbox->box, vname, flags, - VIRTUAL_INDEX_PREFIX); + index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); mbox->storage = storage; mbox->virtual_ext_id = (uint32_t)-1; diff -r f423090a363a -r 15dfd6475f64 src/plugins/virtual/virtual-storage.h --- a/src/plugins/virtual/virtual-storage.h Mon Oct 29 18:32:40 2012 +0200 +++ b/src/plugins/virtual/virtual-storage.h Mon Oct 29 19:52:21 2012 +0200 @@ -7,7 +7,6 @@ #define VIRTUAL_STORAGE_NAME "virtual" #define VIRTUAL_SUBSCRIPTION_FILE_NAME ".virtual-subscriptions" #define VIRTUAL_CONFIG_FNAME "dovecot-virtual" -#define VIRTUAL_INDEX_PREFIX "dovecot.index" #define VIRTUAL_CONTEXT(obj) \ MODULE_CONTEXT(obj, virtual_storage_module) From dovecot at dovecot.org Mon Oct 29 19:54:56 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 19:54:56 +0200 Subject: dovecot-2.2: lib-storage: Renamed mail_storage_set_index_error()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5479887eb461 changeset: 15342:5479887eb461 user: Timo Sirainen date: Mon Oct 29 19:54:50 2012 +0200 description: lib-storage: Renamed mail_storage_set_index_error() to mailbox_set_index_error() diffstat: src/lib-storage/index/cydir/cydir-sync.c | 4 ++-- src/lib-storage/index/dbox-common/dbox-storage.c | 2 +- src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 +++--- src/lib-storage/index/dbox-multi/mdbox-storage.c | 2 +- src/lib-storage/index/dbox-multi/mdbox-sync.c | 4 ++-- src/lib-storage/index/dbox-single/sdbox-storage.c | 2 +- src/lib-storage/index/dbox-single/sdbox-sync.c | 4 ++-- src/lib-storage/index/imapc/imapc-mailbox.c | 4 ++-- src/lib-storage/index/imapc/imapc-storage.c | 2 +- src/lib-storage/index/imapc/imapc-sync.c | 4 ++-- src/lib-storage/index/index-storage.c | 4 ++-- src/lib-storage/index/index-sync-pvt.c | 6 +++--- src/lib-storage/index/index-sync.c | 2 +- src/lib-storage/index/index-transaction.c | 2 +- src/lib-storage/index/maildir/maildir-sync-index.c | 4 ++-- src/lib-storage/index/maildir/maildir-sync.c | 4 ++-- src/lib-storage/index/mbox/mbox-mail.c | 2 +- src/lib-storage/index/mbox/mbox-sync.c | 8 ++++---- src/lib-storage/index/pop3c/pop3c-sync.c | 4 ++-- src/lib-storage/index/raw/raw-sync.c | 4 ++-- src/lib-storage/mail-storage-private.h | 2 +- src/lib-storage/mail-storage.c | 4 ++-- src/plugins/virtual/virtual-sync.c | 4 ++-- 23 files changed, 42 insertions(+), 42 deletions(-) diffs (truncated from 447 to 300 lines): diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/cydir/cydir-sync.c --- a/src/lib-storage/index/cydir/cydir-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/cydir/cydir-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -119,7 +119,7 @@ sync_flags); if (ret <= 0) { if (ret < 0) - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); i_free(ctx); *ctx_r = NULL; return ret; @@ -138,7 +138,7 @@ *_ctx = NULL; if (success) { if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { - mail_storage_set_index_error(&ctx->mbox->box); + mailbox_set_index_error(&ctx->mbox->box); ret = -1; } } else { diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Mon Oct 29 19:54:50 2012 +0200 @@ -282,7 +282,7 @@ ret = mail_index_sync_begin(box->index, &sync_ctx, &view, &trans, 0); if (ret <= 0) { i_assert(ret != 0); - mail_storage_set_index_error(box); + mailbox_set_index_error(box); return -1; } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Mon Oct 29 19:54:50 2012 +0200 @@ -507,7 +507,7 @@ MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES); if (ret <= 0) { i_assert(ret != 0); - mail_storage_set_index_error(box); + mailbox_set_index_error(box); mailbox_free(&box); return -1; } @@ -518,7 +518,7 @@ index_index_rebuild_deinit(&rebuild_ctx, dbox_get_uidvalidity_next); if (mail_index_sync_commit(&sync_ctx) < 0) { - mail_storage_set_index_error(box); + mailbox_set_index_error(box); ret = -1; } @@ -674,7 +674,7 @@ &ctx->prev_msg.trans, 0); if (ret <= 0) { i_assert(ret != 0); - mail_storage_set_index_error(box); + mailbox_set_index_error(box); mailbox_free(&box); return -1; } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-multi/mdbox-storage.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Oct 29 19:54:50 2012 +0200 @@ -304,7 +304,7 @@ mdbox_update_header(mbox, trans, update); if (new_trans != NULL) { if (mail_index_transaction_commit(&new_trans) < 0) { - mail_storage_set_index_error(box); + mailbox_set_index_error(box); return -1; } } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-multi/mdbox-sync.c --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -197,7 +197,7 @@ if (mail_index_reset_fscked(mbox->box.index)) mdbox_storage_set_corrupted(mbox->storage); if (ret < 0) { - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); return -1; } if (ret == 0) { @@ -300,7 +300,7 @@ if (success) { if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { - mail_storage_set_index_error(&ctx->mbox->box); + mailbox_set_index_error(&ctx->mbox->box); ret = -1; } } else { diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-single/sdbox-storage.c --- a/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Mon Oct 29 19:54:50 2012 +0200 @@ -235,7 +235,7 @@ sdbox_update_header(mbox, trans, update); if (new_trans != NULL) { if (mail_index_transaction_commit(&new_trans) < 0) { - mail_storage_set_index_error(box); + mailbox_set_index_error(box); return -1; } } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -213,7 +213,7 @@ sdbox_set_mailbox_corrupted(&mbox->box); if (ret <= 0) { if (ret < 0) - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); array_free(&ctx->expunged_uids); i_free(ctx); *ctx_r = NULL; @@ -264,7 +264,7 @@ if (success) { mail_index_view_ref(ctx->sync_view); if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { - mail_storage_set_index_error(&ctx->mbox->box); + mailbox_set_index_error(&ctx->mbox->box); ret = -1; } else { dbox_sync_expunge_files(ctx); diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/imapc/imapc-mailbox.c --- a/src/lib-storage/index/imapc/imapc-mailbox.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Mon Oct 29 19:54:50 2012 +0200 @@ -70,7 +70,7 @@ array_clear(&mbox->delayed_expunged_uids); ret = mail_index_transaction_commit(&trans); if (ret < 0) - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); return ret; } @@ -85,7 +85,7 @@ mail_index_view_close(&mbox->delayed_sync_view); if (mbox->delayed_sync_trans != NULL) { if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) { - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); ret = -1; } *changes_r = TRUE; diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Mon Oct 29 19:54:50 2012 +0200 @@ -523,7 +523,7 @@ mail_index_view_close(&mbox->delayed_sync_view); if (mbox->delayed_sync_trans != NULL) { if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); } if (mbox->sync_view != NULL) mail_index_view_close(&mbox->sync_view); diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/imapc/imapc-sync.c --- a/src/lib-storage/index/imapc/imapc-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/imapc/imapc-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -377,7 +377,7 @@ sync_flags); if (ret <= 0) { if (ret < 0) - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); i_free(ctx); *ctx_r = NULL; return ret; @@ -412,7 +412,7 @@ *_ctx = NULL; if (ret == 0) { if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { - mail_storage_set_index_error(&ctx->mbox->box); + mailbox_set_index_error(&ctx->mbox->box); ret = -1; } } else { diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/index-storage.c Mon Oct 29 19:54:50 2012 +0200 @@ -239,7 +239,7 @@ if (ret <= 0 || move_to_memory) { if ((index_flags & MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY) != 0) { i_assert(ret <= 0); - mail_storage_set_index_error(box); + mailbox_set_index_error(box); return -1; } @@ -469,7 +469,7 @@ } if ((ret = mail_index_transaction_commit(&trans)) < 0) - mail_storage_set_index_error(box); + mailbox_set_index_error(box); mail_index_view_close(&view); return ret; } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/index-sync-pvt.c --- a/src/lib-storage/index/index-sync-pvt.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/index-sync-pvt.c Mon Oct 29 19:54:50 2012 +0200 @@ -119,7 +119,7 @@ /* open a view for the latest version of the index */ if (mail_index_refresh(box->index) < 0) { - mail_storage_set_index_error(box); + mailbox_set_index_error(box); return -1; } view_shared = mail_index_view_open(box->index); @@ -140,7 +140,7 @@ if (mail_index_sync_begin(box->index_pvt, &sync_ctx, &view_pvt, &trans_pvt, 0) < 0) { - mail_storage_set_index_error(box); + mailbox_set_index_error(box); mail_index_view_close(&view_shared); return -1; } @@ -198,7 +198,7 @@ } if ((ret = mail_index_sync_commit(&sync_ctx)) < 0) - mail_storage_set_index_error(box); + mailbox_set_index_error(box); mail_index_view_close(&view_shared); return ret; } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -369,7 +369,7 @@ if (ctx->sync_ctx != NULL) { if (mail_index_view_sync_commit(&ctx->sync_ctx, &delayed_expunges) < 0) { - mail_storage_set_index_error(_ctx->box); + mailbox_set_index_error(_ctx->box); ret = -1; } } diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/index-transaction.c --- a/src/lib-storage/index/index-transaction.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/index-transaction.c Mon Oct 29 19:54:50 2012 +0200 @@ -40,7 +40,7 @@ t->super.rollback(index_trans); else { if (t->super.commit(index_trans, result_r) < 0) { - mail_storage_set_index_error(t->box); + mailbox_set_index_error(t->box); ret = -1; } else if (result_r->commit_size > 0) { /* something was written to the transaction log */ diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/maildir/maildir-sync-index.c --- a/src/lib-storage/index/maildir/maildir-sync-index.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Mon Oct 29 19:54:50 2012 +0200 @@ -229,7 +229,7 @@ if (mail_index_sync_begin(_box->index, &sync_ctx, &view, &trans, sync_flags) < 0) { - mail_storage_set_index_error(_box); + mailbox_set_index_error(_box); return -1; } @@ -328,7 +328,7 @@ start a second index sync and crash. */ mbox->syncing_commit = TRUE; if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); ret = -1; } mbox->syncing_commit = FALSE; diff -r 15dfd6475f64 -r 5479887eb461 src/lib-storage/index/maildir/maildir-sync.c --- a/src/lib-storage/index/maildir/maildir-sync.c Mon Oct 29 19:52:21 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-sync.c Mon Oct 29 19:54:50 2012 +0200 @@ -569,7 +569,7 @@ int maildir_sync_header_refresh(struct maildir_mailbox *mbox) { if (mail_index_refresh(mbox->box.index) < 0) { - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); return -1; } maildir_sync_get_header(mbox); @@ -1006,7 +1006,7 @@ sync_ctx = mail_index_view_sync_begin(mbox->flags_view, MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) { - mail_storage_set_index_error(&mbox->box); + mailbox_set_index_error(&mbox->box); return -1; } /* make sure the map stays in private memory */ From dovecot at dovecot.org Mon Oct 29 20:09:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:09:54 +0200 Subject: dovecot-2.2: lib-storage: Added mail_storage_service_save_userdb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fec539656811 changeset: 15343:fec539656811 user: Timo Sirainen date: Mon Oct 29 20:06:53 2012 +0200 description: lib-storage: Added mail_storage_service_save_userdb_fields() diffstat: src/lib-storage/mail-storage-service.c | 23 ++++++++++++++++++++++- src/lib-storage/mail-storage-service.h | 4 ++++ 2 files changed, 26 insertions(+), 1 deletions(-) diffs (68 lines): diff -r 5479887eb461 -r fec539656811 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Mon Oct 29 19:54:50 2012 +0200 +++ b/src/lib-storage/mail-storage-service.c Mon Oct 29 20:06:53 2012 +0200 @@ -60,6 +60,9 @@ const char *set_cache_module, *set_cache_service; struct master_service_settings_cache *set_cache; + pool_t userdb_next_pool; + const char *const **userdb_next_fieldsp; + unsigned int debug:1; unsigned int log_initialized:1; }; @@ -991,7 +994,13 @@ /* load global plugins */ mail_storage_service_load_modules(ctx, user_info, user_set); - temp_pool = pool_alloconly_create("userdb lookup", 2048); + if (ctx->userdb_next_pool == NULL) + temp_pool = pool_alloconly_create("userdb lookup", 2048); + else { + temp_pool = ctx->userdb_next_pool; + ctx->userdb_next_pool = NULL; + pool_ref(temp_pool); + } if ((flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) != 0) { ret = service_auth_userdb_lookup(ctx, input, temp_pool, &username, &userdb_fields, @@ -1001,6 +1010,7 @@ pool_unref(&user_pool); return ret; } + *ctx->userdb_next_fieldsp = userdb_fields; } else { userdb_fields = input->userdb_fields; } @@ -1055,6 +1065,17 @@ return ret; } +void mail_storage_service_save_userdb_fields(struct mail_storage_service_ctx *ctx, + pool_t pool, const char *const **userdb_fields_r) +{ + i_assert(pool != NULL); + i_assert(userdb_fields_r != NULL); + + ctx->userdb_next_pool = pool; + ctx->userdb_next_fieldsp = userdb_fields_r; + *userdb_fields_r = NULL; +} + int mail_storage_service_next(struct mail_storage_service_ctx *ctx, struct mail_storage_service_user *user, struct mail_user **mail_user_r) diff -r 5479887eb461 -r fec539656811 src/lib-storage/mail-storage-service.h --- a/src/lib-storage/mail-storage-service.h Mon Oct 29 19:54:50 2012 +0200 +++ b/src/lib-storage/mail-storage-service.h Mon Oct 29 20:06:53 2012 +0200 @@ -81,6 +81,10 @@ const struct mail_storage_service_input *input, struct mail_storage_service_user **user_r, const char **error_r); +/* The next mail_storage_service_lookup() will save the userdb fields into the + given pointer, allocated from the given pool. */ +void mail_storage_service_save_userdb_fields(struct mail_storage_service_ctx *ctx, + pool_t pool, const char *const **userdb_fields_r); /* Returns 0 if ok, -1 if fatal error, -2 if error is user-specific. */ int mail_storage_service_next(struct mail_storage_service_ctx *ctx, struct mail_storage_service_user *user, From dovecot at dovecot.org Mon Oct 29 20:09:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:09:54 +0200 Subject: dovecot-2.2: doveadm user -m: Show all returned userdb fields. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55422e122c27 changeset: 15344:55422e122c27 user: Timo Sirainen date: Mon Oct 29 20:07:13 2012 +0200 description: doveadm user -m: Show all returned userdb fields. diffstat: src/doveadm/doveadm-auth.c | 30 ++++++++++++++++++++++++++++-- 1 files changed, 28 insertions(+), 2 deletions(-) diffs (59 lines): diff -r fec539656811 -r 55422e122c27 src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Oct 29 20:06:53 2012 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Oct 29 20:07:13 2012 +0200 @@ -303,7 +303,9 @@ struct mail_storage_service_user *service_user; struct mail_user *user; const struct mail_storage_settings *mail_set; - const char *error; + const char *key, *value, *error, *const *userdb_fields; + unsigned int i; + pool_t pool; int ret; memset(&service_input, 0, sizeof(service_input)); @@ -315,10 +317,16 @@ service_input.remote_ip = input->info.remote_ip; service_input.remote_port = input->info.remote_port; + pool = pool_alloconly_create("userdb fields", 1024); + mail_storage_service_save_userdb_fields(storage_service, pool, + &userdb_fields); + if ((ret = mail_storage_service_lookup_next(storage_service, &service_input, &service_user, &user, - &error)) <= 0) + &error)) <= 0) { + pool_unref(&pool); return ret == 0 ? 0 : -1; + } if (show_field == NULL) { doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); @@ -333,8 +341,26 @@ mail_set = mail_user_set_get_storage_set(user); cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); + if (userdb_fields != NULL) { + for (i = 0; userdb_fields[i] != NULL; i++) { + value = strchr(userdb_fields[i], '='); + if (value != NULL) + key = t_strdup_until(userdb_fields[i], value++); + else { + key = userdb_fields[i]; + value = ""; + } + if (strcmp(key, "uid") != 0 && + strcmp(key, "gid") != 0 && + strcmp(key, "home") != 0 && + strcmp(key, "mail") != 0) + cmd_user_mail_input_field(key, value, show_field); + } + } + mail_user_unref(&user); mail_storage_service_user_free(&service_user); + pool_unref(&pool); return 1; } From dovecot at dovecot.org Mon Oct 29 20:09:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:09:54 +0200 Subject: dovecot-2.2: doveadm user: Removed -m parameter and made it defa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8e57d4a1cd19 changeset: 15345:8e57d4a1cd19 user: Timo Sirainen date: Mon Oct 29 20:09:42 2012 +0200 description: doveadm user: Removed -m parameter and made it default. Added -u for old functionality. -u meaning "userdb lookup only". diffstat: src/doveadm/doveadm-auth.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (57 lines): diff -r 55422e122c27 -r 8e57d4a1cd19 src/doveadm/doveadm-auth.c --- a/src/doveadm/doveadm-auth.c Mon Oct 29 20:07:13 2012 +0200 +++ b/src/doveadm/doveadm-auth.c Mon Oct 29 20:09:42 2012 +0200 @@ -371,13 +371,13 @@ const char *show_field = NULL; struct mail_storage_service_ctx *storage_service = NULL; unsigned int i; - bool have_wildcards, mail_fields = FALSE, first = TRUE; + bool have_wildcards, userdb_only = FALSE, first = TRUE; int c, ret; memset(&input, 0, sizeof(input)); input.info.service = "doveadm"; - while ((c = getopt(argc, argv, "a:f:mx:")) > 0) { + while ((c = getopt(argc, argv, "a:f:ux:")) > 0) { switch (c) { case 'a': auth_socket_path = optarg; @@ -385,8 +385,8 @@ case 'f': show_field = optarg; break; - case 'm': - mail_fields = TRUE; + case 'u': + userdb_only = TRUE; break; case 'x': auth_user_info_parse(&input.info, optarg); @@ -413,7 +413,7 @@ return; } - if (mail_fields) { + if (!userdb_only) { storage_service = mail_storage_service_init(master_service, NULL, MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP); @@ -425,7 +425,7 @@ else putchar('\n'); - ret = mail_fields ? + ret = !userdb_only ? cmd_user_mail_input(storage_service, &input, show_field) : cmd_user_input(auth_socket_path, &input, show_field); switch (ret) { @@ -447,7 +447,7 @@ { cmd_auth_cache_flush, "auth cache flush", "[-a ] [ [...]]" }, { cmd_user, "user", - "[-a ] [-x ] [-f field] [-m] [...]" } + "[-a ] [-x ] [-f field] [-u] [...]" } }; static void auth_cmd_help(doveadm_command_t *cmd) From dovecot at dovecot.org Mon Oct 29 20:17:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:17:18 +0200 Subject: dovecot-2.2: imap: imap_id_send default changed to send name=Dov... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bf834b3c5663 changeset: 15346:bf834b3c5663 user: Timo Sirainen date: Mon Oct 29 20:17:07 2012 +0200 description: imap: imap_id_send default changed to send name=Dovecot to client. diffstat: src/imap-login/imap-login-settings.c | 2 +- src/imap/imap-settings.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 8e57d4a1cd19 -r bf834b3c5663 src/imap-login/imap-login-settings.c --- a/src/imap-login/imap-login-settings.c Mon Oct 29 20:09:42 2012 +0200 +++ b/src/imap-login/imap-login-settings.c Mon Oct 29 20:17:07 2012 +0200 @@ -63,7 +63,7 @@ static const struct imap_login_settings imap_login_default_settings = { .imap_capability = "", - .imap_id_send = "", + .imap_id_send = "name *", .imap_id_log = "" }; diff -r 8e57d4a1cd19 -r bf834b3c5663 src/imap/imap-settings.c --- a/src/imap/imap-settings.c Mon Oct 29 20:09:42 2012 +0200 +++ b/src/imap/imap-settings.c Mon Oct 29 20:17:07 2012 +0200 @@ -87,7 +87,7 @@ .imap_capability = "", .imap_client_workarounds = "", .imap_logout_format = "in=%i out=%o", - .imap_id_send = "", + .imap_id_send = "name *", .imap_id_log = "", .imap_urlauth_host = "", From dovecot at dovecot.org Mon Oct 29 20:34:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:34:42 +0200 Subject: dovecot-2.2: iostream-rawlog: Avoid crashing if write() to rawlo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bab7bbc0c153 changeset: 15347:bab7bbc0c153 user: Timo Sirainen date: Mon Oct 29 20:34:31 2012 +0200 description: iostream-rawlog: Avoid crashing if write() to rawlog fails. diffstat: src/lib/iostream-rawlog.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r bf834b3c5663 -r bab7bbc0c153 src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Mon Oct 29 20:17:07 2012 +0200 +++ b/src/lib/iostream-rawlog.c Mon Oct 29 20:34:31 2012 +0200 @@ -96,7 +96,7 @@ return; } - do { + while (rstream->rawlog_fd != -1 && size > 0) { p = memchr(data, '\n', size); if (p != NULL) { line_ends = TRUE; @@ -118,7 +118,7 @@ data += pos; size -= pos; - } while (size > 0); + } } void iostream_rawlog_close(struct rawlog_iostream *rstream) From dovecot at dovecot.org Mon Oct 29 20:42:10 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:42:10 +0200 Subject: dovecot-2.2: layout=index: Fixed a crash in mailbox_rename() rac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9050920ce827 changeset: 15348:9050920ce827 user: Timo Sirainen date: Mon Oct 29 20:42:00 2012 +0200 description: layout=index: Fixed a crash in mailbox_rename() race condition. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r bab7bbc0c153 -r 9050920ce827 src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Mon Oct 29 20:34:31 2012 +0200 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Mon Oct 29 20:42:00 2012 +0200 @@ -483,6 +483,12 @@ (void)mailbox_list_index_sync_end(&sync_ctx, FALSE); return -1; } + if (ret == 0) { + (void)mailbox_list_index_sync_end(&sync_ctx, FALSE); + mailbox_list_set_error(&list->list, MAIL_ERROR_NOTFOUND, + T_MAIL_ERR_MAILBOX_NOT_FOUND(oldname)); + return -1; + } if (!mail_index_lookup_seq(sync_ctx->view, oldnode->uid, &oldseq)) i_panic("mailbox list index: lost uid=%u", oldnode->uid); From dovecot at dovecot.org Mon Oct 29 20:45:54 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:45:54 +0200 Subject: dovecot-2.2: maildir: Fixed potential crash when maildir suddenl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/755b03fb6fce changeset: 15349:755b03fb6fce user: Timo Sirainen date: Mon Oct 29 20:45:41 2012 +0200 description: maildir: Fixed potential crash when maildir suddenly loses some directories. diffstat: src/lib-storage/index/maildir/maildir-util.c | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) diffs (28 lines): diff -r 9050920ce827 -r 755b03fb6fce src/lib-storage/index/maildir/maildir-util.c --- a/src/lib-storage/index/maildir/maildir-util.c Mon Oct 29 20:42:00 2012 +0200 +++ b/src/lib-storage/index/maildir/maildir-util.c Mon Oct 29 20:45:41 2012 +0200 @@ -206,7 +206,7 @@ enum mailbox_list_path_type types[N_ELEMENTS(subdirs) + 2]; struct stat st; const char *path; - unsigned int i; + unsigned int i, count; /* @UNSAFE: get a list of directories we want to create */ for (i = 0; i < N_ELEMENTS(subdirs); i++) { @@ -222,11 +222,12 @@ types[i] = MAILBOX_LIST_PATH_TYPE_INDEX; dirs[i++] = path; } - i_assert(i <= N_ELEMENTS(dirs)); + count = i; + i_assert(count <= N_ELEMENTS(dirs)); - for (i = 0; i < N_ELEMENTS(dirs); i++) { + for (i = 0; i < count; i++) { path = dirs[i]; - if (path == NULL || stat(path, &st) == 0) + if (stat(path, &st) == 0) continue; if (errno != ENOENT) { mail_storage_set_critical(box->storage, From dovecot at dovecot.org Mon Oct 29 20:48:38 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 20:48:38 +0200 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bb095315d025 changeset: 15350:bb095315d025 user: Timo Sirainen date: Mon Oct 29 20:48:08 2012 +0200 description: Make static analyzer happier. diffstat: src/director/director-connection.c | 2 +- src/director/director-test.c | 1 + src/imap/cmd-urlfetch.c | 4 ++-- src/lib-imap-urlauth/imap-urlauth-fetch.c | 3 +-- src/lib-index/mail-cache-transaction.c | 1 + src/lib-index/mail-transaction-log.c | 2 +- src/lib-mail/message-parser.c | 6 ++++-- 7 files changed, 11 insertions(+), 8 deletions(-) diffs (104 lines): diff -r 755b03fb6fce -r bb095315d025 src/director/director-connection.c --- a/src/director/director-connection.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/director/director-connection.c Mon Oct 29 20:48:08 2012 +0200 @@ -708,11 +708,11 @@ ourself. */ *host_r = NULL; } else { + *host_r = host; if (seq <= host->last_seq) { /* already seen this */ return 1; } - *host_r = host; host->last_seq = seq; } return 0; diff -r 755b03fb6fce -r bb095315d025 src/director/director-test.c --- a/src/director/director-test.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/director/director-test.c Mon Oct 29 20:48:08 2012 +0200 @@ -526,6 +526,7 @@ i_assert(conn != NULL); i++; } + i_assert(conn != NULL); director_connection_destroy(&conn); } } diff -r 755b03fb6fce -r bb095315d025 src/imap/cmd-urlfetch.c --- a/src/imap/cmd-urlfetch.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/imap/cmd-urlfetch.c Mon Oct 29 20:48:08 2012 +0200 @@ -72,7 +72,7 @@ struct client *client = cmd->client; struct cmd_urlfetch_context *ctx = (struct cmd_urlfetch_context *)cmd->context; - int ret = 1; + int ret; /* are we in the middle of an urlfetch literal? */ if (ctx->input == NULL) @@ -87,7 +87,7 @@ /* transfer literal to client */ o_stream_set_max_buffer_size(client->output, 0); - ret = o_stream_send_istream(client->output, ctx->input); + (void)o_stream_send_istream(client->output, ctx->input); o_stream_set_max_buffer_size(client->output, (size_t)-1); if (ctx->input->v_offset == ctx->size) { diff -r 755b03fb6fce -r bb095315d025 src/lib-imap-urlauth/imap-urlauth-fetch.c --- a/src/lib-imap-urlauth/imap-urlauth-fetch.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/lib-imap-urlauth/imap-urlauth-fetch.c Mon Oct 29 20:48:08 2012 +0200 @@ -198,8 +198,7 @@ } /* if requested, read the message part the URL points to */ - mpresult.size = 0; - mpresult.input = NULL; + memset(&mpresult, 0, sizeof(mpresult)); if (success && ((url_flags & IMAP_URLAUTH_FETCH_FLAG_BODY) != 0 || (url_flags & IMAP_URLAUTH_FETCH_FLAG_BINARY) != 0)) { ret = imap_msgpart_url_read_part(mpurl, &mpresult, &error); diff -r 755b03fb6fce -r bb095315d025 src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/lib-index/mail-cache-transaction.c Mon Oct 29 20:48:08 2012 +0200 @@ -335,6 +335,7 @@ return -1; /* first write the actual data to cache file */ + i_assert(ctx->cache_data != NULL); i_assert(ctx->last_rec_pos <= ctx->cache_data->used); if (mail_cache_append(ctx->cache, ctx->cache_data->data, ctx->last_rec_pos, &write_offset) < 0) diff -r 755b03fb6fce -r bb095315d025 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/lib-index/mail-transaction-log.c Mon Oct 29 20:48:08 2012 +0200 @@ -452,7 +452,7 @@ if (ret == 0 && log->head == file) { /* success */ - lock_secs = file->lock_created - lock_wait_started; + lock_secs = log->head->lock_created - lock_wait_started; break; } diff -r 755b03fb6fce -r bb095315d025 src/lib-mail/message-parser.c --- a/src/lib-mail/message-parser.c Mon Oct 29 20:45:41 2012 +0200 +++ b/src/lib-mail/message-parser.c Mon Oct 29 20:48:08 2012 +0200 @@ -690,12 +690,14 @@ static int preparsed_parse_prologue_more(struct message_parser_ctx *ctx, struct message_block *block_r) { - uoff_t end_offset = ctx->part->children->physical_pos; - uoff_t boundary_min_start; + uoff_t boundary_min_start, end_offset; const unsigned char *cur; bool full; int ret; + i_assert(ctx->part->children != NULL); + end_offset = ctx->part->children->physical_pos; + if ((ret = message_parser_read_more(ctx, block_r, &full)) <= 0) return ret; From dovecot at dovecot.org Mon Oct 29 21:31:28 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 21:31:28 +0200 Subject: dovecot-2.2: iostream-rawlog: Avoid crashing more if write() to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ce342fd3cae7 changeset: 15351:ce342fd3cae7 user: Timo Sirainen date: Mon Oct 29 21:28:45 2012 +0200 description: iostream-rawlog: Avoid crashing more if write() to rawlog fails. diffstat: src/lib/iostream-rawlog.c | 38 ++++++++++++++++++++++++-------------- 1 files changed, 24 insertions(+), 14 deletions(-) diffs (88 lines): diff -r bb095315d025 -r ce342fd3cae7 src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Mon Oct 29 20:48:08 2012 +0200 +++ b/src/lib/iostream-rawlog.c Mon Oct 29 21:28:45 2012 +0200 @@ -20,20 +20,22 @@ #define RAWLOG_MAX_LINE_LEN 8192 -static void +static int rawlog_write(struct rawlog_iostream *rstream, const void *data, size_t size) { if (rstream->rawlog_fd == -1) - return; + return -1; if (write_full(rstream->rawlog_fd, data, size) < 0) { i_error("rawlog_istream.write(%s) failed: %m", rstream->rawlog_path); iostream_rawlog_close(rstream); + return -1; } + return 0; } -static void +static int rawlog_write_timestamp(struct rawlog_iostream *rstream, bool line_ends) { unsigned char data[MAX_INT_STRLEN + 6 + 1 + 3]; @@ -48,7 +50,7 @@ str_append_c(&buf, line_ends ? ':' : '>'); str_append_c(&buf, ' '); } - rawlog_write(rstream, buf.data, buf.used); + return rawlog_write(rstream, buf.data, buf.used); } void iostream_rawlog_init(struct rawlog_iostream *rstream, @@ -66,18 +68,23 @@ { size_t i, start; - if (!rstream->line_continued) - rawlog_write_timestamp(rstream, TRUE); + if (!rstream->line_continued) { + if (rawlog_write_timestamp(rstream, TRUE) < 0) + return; + } for (start = 0, i = 1; i < size; i++) { if (data[i-1] == '\n') { - rawlog_write(rstream, data + start, i - start); - rawlog_write_timestamp(rstream, TRUE); + if (rawlog_write(rstream, data + start, i - start) < 0 || + rawlog_write_timestamp(rstream, TRUE) < 0) + return; start = i; } } - if (start != size) - rawlog_write(rstream, data + start, size - start); + if (start != size) { + if (rawlog_write(rstream, data + start, size - start) < 0) + return; + } rstream->line_continued = data[size-1] != '\n'; } @@ -109,12 +116,15 @@ pos = size; } - rawlog_write_timestamp(rstream, line_ends); + if (rawlog_write_timestamp(rstream, line_ends) < 0) + break; if (rstream->buffer->used > 0) { - rawlog_write(rstream, rstream->buffer->data, - rstream->buffer->used); + if (rawlog_write(rstream, rstream->buffer->data, + rstream->buffer->used) < 0) + break; } - rawlog_write(rstream, data, pos); + if (rawlog_write(rstream, data, pos) < 0) + break; data += pos; size -= pos; From dovecot at dovecot.org Mon Oct 29 21:31:44 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 21:31:44 +0200 Subject: dovecot-2.2: Make static analyzer happier. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a40d29049b2 changeset: 15352:3a40d29049b2 user: Timo Sirainen date: Mon Oct 29 21:31:39 2012 +0200 description: Make static analyzer happier. diffstat: src/lib-dict/dict-file.c | 5 +++-- src/lib-index/mail-transaction-log.c | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diffs (31 lines): diff -r ce342fd3cae7 -r 3a40d29049b2 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Mon Oct 29 21:28:45 2012 +0200 +++ b/src/lib-dict/dict-file.c Mon Oct 29 21:31:39 2012 +0200 @@ -244,11 +244,12 @@ { struct file_dict_iterate_context *ctx = (struct file_dict_iterate_context *)_ctx; - struct file_dict *dict = (struct file_dict *)_ctx->dict; const struct file_dict_iterate_path *path; char *key, *value; - while (hash_table_iterate(ctx->iter, dict->hash, &key, &value)) { + while (hash_table_iterate(ctx->iter, + ((struct file_dict *)_ctx->dict)->hash, + &key, &value)) { path = file_dict_iterate_find_path(ctx, key); if (path == NULL) continue; diff -r ce342fd3cae7 -r 3a40d29049b2 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Mon Oct 29 21:28:45 2012 +0200 +++ b/src/lib-index/mail-transaction-log.c Mon Oct 29 21:31:39 2012 +0200 @@ -452,7 +452,8 @@ if (ret == 0 && log->head == file) { /* success */ - lock_secs = log->head->lock_created - lock_wait_started; + i_assert(file != NULL); + lock_secs = file->lock_created - lock_wait_started; break; } From dovecot at dovecot.org Mon Oct 29 21:42:49 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 21:42:49 +0200 Subject: dovecot-2.2: autocreate plugin logs now a "deprecated" warning a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/275ca1a03cdd changeset: 15353:275ca1a03cdd user: Timo Sirainen date: Mon Oct 29 21:42:32 2012 +0200 description: autocreate plugin logs now a "deprecated" warning at startup diffstat: src/plugins/autocreate/autocreate-plugin.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 3a40d29049b2 -r 275ca1a03cdd src/plugins/autocreate/autocreate-plugin.c --- a/src/plugins/autocreate/autocreate-plugin.c Mon Oct 29 21:31:39 2012 +0200 +++ b/src/plugins/autocreate/autocreate-plugin.c Mon Oct 29 21:42:32 2012 +0200 @@ -93,6 +93,7 @@ void autocreate_plugin_init(struct module *module) { + i_warning("autocreate plugin is deprecated, use mailbox { auto } setting instead"); mail_storage_hooks_add(module, &autocreate_mail_storage_hooks); } From dovecot at dovecot.org Mon Oct 29 21:58:29 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 21:58:29 +0200 Subject: dovecot-2.2: lib-storage: Use const pointers for mail_namespace.... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/62519d9334a5 changeset: 15354:62519d9334a5 user: Timo Sirainen date: Mon Oct 29 21:54:51 2012 +0200 description: lib-storage: Use const pointers for mail_namespace.(set|unexpanded_set) diffstat: src/lib-storage/mail-namespace.h | 2 +- src/plugins/autocreate/autocreate-plugin.c | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diffs (50 lines): diff -r 275ca1a03cdd -r 62519d9334a5 src/lib-storage/mail-namespace.h --- a/src/lib-storage/mail-namespace.h Mon Oct 29 21:42:32 2012 +0200 +++ b/src/lib-storage/mail-namespace.h Mon Oct 29 21:54:51 2012 +0200 @@ -70,7 +70,7 @@ /* FIXME: we should support multiple storages in one namespace */ struct mail_storage *storage; - struct mail_namespace_settings *set, *unexpanded_set; + const struct mail_namespace_settings *set, *unexpanded_set; const struct mail_storage_settings *mail_set; unsigned int special_use_mailboxes:1; diff -r 275ca1a03cdd -r 62519d9334a5 src/plugins/autocreate/autocreate-plugin.c --- a/src/plugins/autocreate/autocreate-plugin.c Mon Oct 29 21:42:32 2012 +0200 +++ b/src/plugins/autocreate/autocreate-plugin.c Mon Oct 29 21:54:51 2012 +0200 @@ -28,6 +28,7 @@ { struct mail_namespace *ns; struct mailbox_settings *set; + struct mail_namespace_settings tmp_ns_set; if (!uni_utf8_str_is_valid(vname)) { i_error("autocreate: Mailbox name isn't valid UTF-8: %s", @@ -42,8 +43,15 @@ return; } - if (!array_is_created(&ns->set->mailboxes)) - p_array_init(&ns->set->mailboxes, user->pool, 16); + if (array_is_created(&ns->set->mailboxes)) + tmp_ns_set.mailboxes = ns->set->mailboxes; + else { + p_array_init(&tmp_ns_set.mailboxes, user->pool, 16); + /* work around ns->set being a const pointer. pretty ugly, but + this plugin is deprecated anyway. */ + memcpy((void *)&ns->set->mailboxes.arr, &tmp_ns_set.mailboxes.arr, + sizeof(ns->set->mailboxes.arr)); + } if (strncmp(vname, ns->prefix, ns->prefix_len) == 0) vname += ns->prefix_len; @@ -53,7 +61,7 @@ set->name = p_strdup(user->pool, vname); set->autocreate = MAILBOX_SET_AUTO_NO; set->special_use = ""; - array_append(&ns->set->mailboxes, &set, 1); + array_append(&tmp_ns_set.mailboxes, &set, 1); } if (subscriptions) set->autocreate = MAILBOX_SET_AUTO_SUBSCRIBE; From dovecot at dovecot.org Mon Oct 29 22:03:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 22:03:36 +0200 Subject: dovecot-2.2: If prefix="" namespace isn't defined, autocreate it... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b52f4852e5f8 changeset: 15355:b52f4852e5f8 user: Timo Sirainen date: Mon Oct 29 22:03:25 2012 +0200 description: If prefix="" namespace isn't defined, autocreate it as an unusable namespace. This avoids having to handle mail_namespace_find() errors all over the place. Instead now the mailbox accesses will simply fail. diffstat: configure.ac | 2 +- src/doveadm/doveadm-mail-copymove.c | 5 -- src/doveadm/doveadm-mail-import.c | 15 +------ src/doveadm/doveadm-mail-mailbox.c | 29 ------------- src/doveadm/doveadm-mail.c | 5 -- src/doveadm/doveadm-mailbox-list-iter.c | 5 +- src/doveadm/dsync/doveadm-dsync.c | 4 - src/doveadm/dsync/dsync-brain.c | 6 -- src/imap/cmd-list.c | 2 - src/imap/imap-commands-util.c | 5 +- src/indexer/master-connection.c | 5 -- src/lib-imap-storage/imap-msgpart-url.c | 5 -- src/lib-lda/mail-deliver.c | 6 -- src/lib-storage/fail-mail-storage.c | 8 ++- src/lib-storage/mail-namespace.c | 47 ++++++++++++++++++++- src/lib-storage/mail-namespace.h | 19 +++++--- src/lib-storage/mail-storage.c | 2 - src/plugins/acl/acl-mailbox-list.c | 4 + src/plugins/acl/doveadm-acl.c | 10 ---- src/plugins/autocreate/autocreate-plugin.c | 2 +- src/plugins/pop3-migration/pop3-migration-plugin.c | 5 -- src/plugins/quota/quota-storage.c | 4 +- src/plugins/snarf/snarf-plugin.c | 5 -- src/plugins/trash/trash-plugin.c | 10 ++-- src/pop3/pop3-client.c | 2 +- 25 files changed, 83 insertions(+), 129 deletions(-) diffs (truncated from 581 to 300 lines): diff -r 62519d9334a5 -r b52f4852e5f8 configure.ac --- a/configure.ac Mon Oct 29 21:54:51 2012 +0200 +++ b/configure.ac Mon Oct 29 22:03:25 2012 +0200 @@ -250,7 +250,7 @@ mail_storages="shared `echo "$withval"|sed 's/,/ /g'`" ], mail_storages="shared mdbox sdbox maildir mbox cydir") AC_SUBST(mail_storages) -mail_storages="$mail_storages imapc_stub pop3c_stub raw" +mail_storages="$mail_storages imapc_stub pop3c_stub raw fail" # drop duplicates duplicates=`(for i in $mail_storages; do echo $i; done)|sort|uniq -d|xargs echo` if test "$duplicates" != ""; then diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/doveadm-mail-copymove.c --- a/src/doveadm/doveadm-mail-copymove.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/doveadm-mail-copymove.c Mon Oct 29 22:03:25 2012 +0200 @@ -103,11 +103,6 @@ int ret = 0; ns = mail_namespace_find(user->namespaces, ctx->destname); - if (ns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for: %s", ctx->destname); - } - destbox = mailbox_alloc(ns->list, ctx->destname, MAILBOX_FLAG_SAVEONLY); if (mailbox_open(destbox) < 0) { i_error("Can't open mailbox '%s': %s", ctx->destname, diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/doveadm-mail-import.c --- a/src/doveadm/doveadm-mail-import.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/doveadm-mail-import.c Mon Oct 29 22:03:25 2012 +0200 @@ -30,21 +30,10 @@ if (*ctx->dest_parent != '\0') { /* prefix destination mailbox name with given parent mailbox */ ns = mail_namespace_find(user->namespaces, ctx->dest_parent); - if (ns == NULL) { - i_error("Can't find namespace for parent mailbox %s", - ctx->dest_parent); - doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_NOTFOUND); - return -1; - } name = t_strdup_printf("%s%c%s", ctx->dest_parent, mail_namespace_get_sep(ns), name); - } - - ns = mail_namespace_find(user->namespaces, name); - if (ns == NULL) { - i_error("Can't find namespace for mailbox %s", name); - doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_NOTFOUND); - return -1; + } else { + ns = mail_namespace_find(user->namespaces, name); } box = mailbox_alloc(ns->list, name, MAILBOX_FLAG_SAVEONLY); diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/doveadm-mail-mailbox.c --- a/src/doveadm/doveadm-mail-mailbox.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/doveadm-mail-mailbox.c Mon Oct 29 22:03:25 2012 +0200 @@ -202,11 +202,6 @@ bool directory = FALSE; ns = mail_namespace_find(user->namespaces, name); - if (ns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for: %s", name); - } - len = strlen(name); if (len > 0 && name[len-1] == mail_namespace_get_sep(ns)) { name = t_strndup(name, len-1); @@ -276,9 +271,6 @@ const char *pattern, *child_name; ns = mail_namespace_find(user->namespaces, name); - if (ns == NULL) - return 0; - pattern = t_strdup_printf("%s%c*", name, mail_namespace_get_sep(ns)); iter = mailbox_list_iter_init(ns->list, pattern, MAILBOX_LIST_ITER_RETURN_NO_FLAGS); @@ -320,13 +312,6 @@ const char *name = *namep; ns = mail_namespace_find(user->namespaces, name); - if (ns == NULL) { - i_error("Can't find namespace for: %s", name); - doveadm_mail_failed_error(_ctx, MAIL_ERROR_NOTFOUND); - ret = -1; - continue; - } - box = mailbox_alloc(ns->list, name, 0); storage = mailbox_get_storage(box); if (mailbox_delete(box) < 0) { @@ -409,16 +394,7 @@ int ret = 0; oldns = mail_namespace_find(user->namespaces, oldname); - if (oldns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for: %s", oldname); - } newns = mail_namespace_find(user->namespaces, newname); - if (newns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for: %s", newname); - } - oldbox = mailbox_alloc(oldns->list, oldname, 0); newbox = mailbox_alloc(newns->list, newname, 0); if (mailbox_rename(oldbox, newbox) < 0) { @@ -484,11 +460,6 @@ const char *name = *namep; ns = mail_namespace_find(user->namespaces, name); - if (ns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for: %s", name); - } - box = mailbox_alloc(ns->list, name, 0); if (mailbox_set_subscribed(box, ctx->ctx.subscriptions) < 0) { i_error("Can't %s mailbox %s: %s", name, diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/doveadm-mail.c Mon Oct 29 22:03:25 2012 +0200 @@ -143,11 +143,6 @@ } ns = mail_namespace_find(user->namespaces, mailbox); - if (ns == NULL) { - i_fatal_status(DOVEADM_EX_NOTFOUND, - "Can't find namespace for mailbox %s", mailbox); - } - return mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_IGNORE_ACLS); } diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/doveadm-mailbox-list-iter.c --- a/src/doveadm/doveadm-mailbox-list-iter.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/doveadm-mailbox-list-iter.c Mon Oct 29 22:03:25 2012 +0200 @@ -173,10 +173,7 @@ iter->info.vname = patterns[iter->pattern_idx++]; iter->info.ns = mail_namespace_find(iter->user->namespaces, iter->info.vname); - if (iter->info.ns != NULL) - return &iter->info; - /* FIXME: maybe fail?.. or just wait for v2.2 to get rid of - this error condition */ + return &iter->info; } while ((info = mailbox_list_iter_next(iter->iter)) != NULL) { diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Oct 29 22:03:25 2012 +0200 @@ -328,10 +328,6 @@ if (ctx->namespace_prefix != NULL) { sync_ns = mail_namespace_find(user->namespaces, ctx->namespace_prefix); - if (sync_ns == NULL) { - i_fatal("Namespace prefix=%s doesn't exist", - ctx->namespace_prefix); - } } if (!ctx->remote) diff -r 62519d9334a5 -r b52f4852e5f8 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Mon Oct 29 22:03:25 2012 +0200 @@ -165,12 +165,6 @@ if (ibc_set->sync_ns_prefix != NULL) { brain->sync_ns = mail_namespace_find(brain->user->namespaces, ibc_set->sync_ns_prefix); - if (brain->sync_ns == NULL) { - i_error("Requested sync namespace prefix=%s doesn't exist", - ibc_set->sync_ns_prefix); - brain->failed = TRUE; - return TRUE; - } } i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); brain->sync_type = ibc_set->sync_type; diff -r 62519d9334a5 -r b52f4852e5f8 src/imap/cmd-list.c --- a/src/imap/cmd-list.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/imap/cmd-list.c Mon Oct 29 22:03:25 2012 +0200 @@ -284,8 +284,6 @@ return patterns; ns = mail_namespace_find(ctx->user->namespaces, ref); - if (ns == NULL) - return patterns; t_array_init(&full_patterns, 16); for (pat = patterns; *pat != NULL; pat++) { diff -r 62519d9334a5 -r b52f4852e5f8 src/imap/imap-commands-util.c --- a/src/imap/imap-commands-util.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/imap/imap-commands-util.c Mon Oct 29 22:03:25 2012 +0200 @@ -28,7 +28,10 @@ } ns = mail_namespace_find(namespaces, str_c(utf8_name)); - if (ns == NULL) { + if ((ns->flags & NAMESPACE_FLAG_AUTOCREATED) != 0 && + ns->prefix_len == 0) { + /* this matched only the autocreated prefix="" namespace. + give a nice human-readable error message */ client_send_tagline(cmd, t_strdup_printf( "NO Client tried to access nonexistent namespace. " "(Mailbox name should probably be prefixed with: %s)", diff -r 62519d9334a5 -r b52f4852e5f8 src/indexer/master-connection.c --- a/src/indexer/master-connection.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/indexer/master-connection.c Mon Oct 29 22:03:25 2012 +0200 @@ -125,11 +125,6 @@ int ret; ns = mail_namespace_find(user->namespaces, mailbox); - if (ns == NULL) { - i_error("Namespace not found for mailbox %s: ", mailbox); - return -1; - } - box = mailbox_alloc(ns->list, mailbox, 0); ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path); if (ret <= 0) { diff -r 62519d9334a5 -r b52f4852e5f8 src/lib-imap-storage/imap-msgpart-url.c --- a/src/lib-imap-storage/imap-msgpart-url.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/lib-imap-storage/imap-msgpart-url.c Mon Oct 29 22:03:25 2012 +0200 @@ -124,11 +124,6 @@ /* find mailbox namespace */ ns = mail_namespace_find(mpurl->user->namespaces, mpurl->mailbox); - if (ns == NULL) { - *error_r = "Nonexistent mailbox namespace"; - *error_code_r = MAIL_ERROR_NOTFOUND; - return 0; - } /* open mailbox */ if (mpurl->selected_box != NULL && diff -r 62519d9334a5 -r b52f4852e5f8 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/lib-lda/mail-deliver.c Mon Oct 29 22:03:25 2012 +0200 @@ -164,12 +164,6 @@ } ns = mail_namespace_find(ctx->user->namespaces, name); - if (ns == NULL) { - *error_str_r = "Unknown namespace"; - *error_r = MAIL_ERROR_PARAMS; - return -1; - } - if (strcmp(name, ns->prefix) == 0 && (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { /* delivering to a namespace prefix means we actually want to diff -r 62519d9334a5 -r b52f4852e5f8 src/lib-storage/fail-mail-storage.c --- a/src/lib-storage/fail-mail-storage.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/lib-storage/fail-mail-storage.c Mon Oct 29 22:03:25 2012 +0200 @@ -19,6 +19,10 @@ return storage; } +static void fail_storage_destroy(struct mail_storage *storage ATTR_UNUSED) +{ +} + static void fail_storage_get_list_settings(const struct mail_namespace *ns ATTR_UNUSED, struct mailbox_list_settings *set) @@ -31,13 +35,13 @@ struct mail_storage fail_storage = { .name = "fail", - .class_flags = 0, + .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT, .v = { NULL, fail_storage_alloc, NULL, - NULL, + fail_storage_destroy, NULL, fail_storage_get_list_settings, NULL, diff -r 62519d9334a5 -r b52f4852e5f8 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Mon Oct 29 21:54:51 2012 +0200 +++ b/src/lib-storage/mail-namespace.c Mon Oct 29 22:03:25 2012 +0200 @@ -12,6 +12,25 @@ #include +static struct mail_namespace_settings prefixless_ns_unexpanded_set = { + .name = "", + .type = "private", + .separator = "", + .prefix = "0", + .location = "0fail::LAYOUT=none", + .alias_for = NULL, + From dovecot at dovecot.org Mon Oct 29 22:42:33 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 22:42:33 +0200 Subject: dovecot-2.2: anvil: Added a comment to remind what the anvil che... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/483199c2ca13 changeset: 15356:483199c2ca13 user: Timo Sirainen date: Mon Oct 29 22:42:28 2012 +0200 description: anvil: Added a comment to remind what the anvil checksums are all about. diffstat: src/anvil/penalty.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (13 lines): diff -r b52f4852e5f8 -r 483199c2ca13 src/anvil/penalty.c --- a/src/anvil/penalty.c Mon Oct 29 22:03:25 2012 +0200 +++ b/src/anvil/penalty.c Mon Oct 29 22:42:28 2012 +0200 @@ -1,5 +1,9 @@ /* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ +/* The idea behind checksums is that the same username+password doesn't + increase the penalty, because it's most likely a user with a misconfigured + account. */ + #include "lib.h" #include "ioloop.h" #include "hash.h" From dovecot at dovecot.org Mon Oct 29 23:01:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 23:01:05 +0200 Subject: dovecot-2.2: Released v2.2.alpha1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f5941f3ac762 changeset: 15357:f5941f3ac762 user: Timo Sirainen date: Mon Oct 29 22:47:32 2012 +0200 description: Released v2.2.alpha1. diffstat: NEWS | 2 ++ TODO | 14 +++++++++++++- configure.ac | 6 +++++- 3 files changed, 20 insertions(+), 2 deletions(-) diffs (53 lines): diff -r 483199c2ca13 -r f5941f3ac762 NEWS --- a/NEWS Mon Oct 29 22:42:28 2012 +0200 +++ b/NEWS Mon Oct 29 22:47:32 2012 +0200 @@ -4,6 +4,8 @@ parent directory if it has setgid-bit set. For full details, see http://wiki2.dovecot.org/SharedMailboxes/Permissions * "doveadm auth" command was renamed to "doveadm auth test" + * IMAP: ID command now advertises server name as Dovecot by default. + It was already trivial to guess this from command replies. + Implemented IMAP MOVE and BINARY extensions + Implemented IMAP CATENATE, URLAUTH and URLAUTH=BINARY extensions diff -r 483199c2ca13 -r f5941f3ac762 TODO --- a/TODO Mon Oct 29 22:42:28 2012 +0200 +++ b/TODO Mon Oct 29 22:47:32 2012 +0200 @@ -10,9 +10,21 @@ imap-urlauth-client.c (AFTER destroying the worker) - special response in the control connection to make the imap-urlauth master wait before starting a new worker - - finish dsync rewrite + - dsync to sync both shared and private message flags + - pop3_lock_session=yes should use its own separate pop3-lock file + - Track highestmodseq always, just don't keep per-message modseqs unless + they're enabled. + - LDA/LMTP: Allow saving one last mail even if it brings user over quota? + - zlib plugin should do caching similar to binary fetches. otherwise + partial fetches from large compressed mails are hopelessly slow. - imaptest: add condstore, qresync tests + - shared user should get settings from userdb extra fields, especially + plugin/quota_rule to get different quota limits for shared mailboxes. + the problem is that user doesn't currently have set_parser available, + and adding it would probably waste memory.. + - auth_debug[_passwords]=yes ability for specific users via doveadm. for + both login-common and auth - settings parsing is horribly bloaty - doveadm: if running via doveadm-server and it fails, say something about error being in the log diff -r 483199c2ca13 -r f5941f3ac762 configure.ac --- a/configure.ac Mon Oct 29 22:42:28 2012 +0200 +++ b/configure.ac Mon Oct 29 22:47:32 2012 +0200 @@ -1,6 +1,10 @@ AC_PREREQ([2.59]) -AC_INIT([Dovecot],[2.2.UNSTABLE],[dovecot at dovecot.org]) + +# Be sure to update ABI version also if anything changes that might require +# recompiling plugins. Most importantly that means if any structs are changed. +AC_INIT([Dovecot],[2.2.alpha1],[dovecot at dovecot.org]) AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv0($PACKAGE_VERSION)", [Dovecot ABI version]) + AC_CONFIG_SRCDIR([src]) AM_INIT_AUTOMAKE([foreign]) From dovecot at dovecot.org Mon Oct 29 23:01:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 23:01:05 +0200 Subject: dovecot-2.2: Added tag 2.2.alpha1 for changeset f5941f3ac762 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9cf290152139 changeset: 15358:9cf290152139 user: Timo Sirainen date: Mon Oct 29 22:47:32 2012 +0200 description: Added tag 2.2.alpha1 for changeset f5941f3ac762 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r f5941f3ac762 -r 9cf290152139 .hgtags --- a/.hgtags Mon Oct 29 22:47:32 2012 +0200 +++ b/.hgtags Mon Oct 29 22:47:32 2012 +0200 @@ -87,3 +87,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 2.1.8 bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 +f5941f3ac7622361634b6cba464da79cc883d1bb 2.2.alpha1 From dovecot at dovecot.org Mon Oct 29 23:01:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 23:01:05 +0200 Subject: dovecot-2.2: Added signature for changeset f5941f3ac762 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ea4cad23d53 changeset: 15359:9ea4cad23d53 user: Timo Sirainen date: Mon Oct 29 22:47:35 2012 +0200 description: Added signature for changeset f5941f3ac762 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 9cf290152139 -r 9ea4cad23d53 .hgsigs --- a/.hgsigs Mon Oct 29 22:47:32 2012 +0200 +++ b/.hgsigs Mon Oct 29 22:47:35 2012 +0200 @@ -50,3 +50,4 @@ 7e5f36fd989d27a2fb48250adbab8fa54ddb6083 0 iEYEABECAAYFAk/yVakACgkQyUhSUUBVismekwCfSEVQjd6fwdChjd53LSt03b4UWKoAoIxd/IjLatTISlHm44iiQwzRKByo bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT +f5941f3ac7622361634b6cba464da79cc883d1bb 0 iEYEABECAAYFAlCO62QACgkQyUhSUUBViskUtQCffWRQpSqaf+iCOipsTWE1D93TwVEAnAhxx1aezuqDVAsjWoYZkrIufO28 From dovecot at dovecot.org Mon Oct 29 23:04:04 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Oct 2012 23:04:04 +0200 Subject: dovecot-2.2: doveadm sync/backup: Added -s parameter to usage st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d29a1bf4aba5 changeset: 15360:d29a1bf4aba5 user: Timo Sirainen date: Mon Oct 29 23:03:59 2012 +0200 description: doveadm sync/backup: Added -s parameter to usage string. diffstat: src/doveadm/dsync/doveadm-dsync.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (17 lines): diff -r 9ea4cad23d53 -r d29a1bf4aba5 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Oct 29 22:47:35 2012 +0200 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Oct 29 23:03:59 2012 +0200 @@ -590,11 +590,11 @@ struct doveadm_mail_cmd cmd_dsync_mirror = { cmd_dsync_alloc, "sync", - "[-dfR] [-l ] [-m ] [-n ] " + "[-dfR] [-l ] [-m ] [-n ] [-s ] " }; struct doveadm_mail_cmd cmd_dsync_backup = { cmd_dsync_backup_alloc, "backup", - "[-dfR] [-l ] [-m ] [-n ] " + "[-dfR] [-l ] [-m ] [-n ] [-s ] " }; struct doveadm_mail_cmd cmd_dsync_server = { cmd_dsync_server_alloc, "dsync-server", &doveadm_mail_cmd_hide From pigeonhole at rename-it.nl Mon Oct 29 23:42:40 2012 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 29 Oct 2012 22:42:40 +0100 Subject: dovecot-2.2-pigeonhole: Renamed configure.in to configure.ac. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/613258f9f541 changeset: 1699:613258f9f541 user: Stephan Bosch date: Mon Oct 29 22:42:34 2012 +0100 description: Renamed configure.in to configure.ac. Apparently automakes in future won't support configure.in anymore. diffstat: configure.ac | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ configure.in | 140 ----------------------------------------------------------- 2 files changed, 140 insertions(+), 140 deletions(-) diffs (288 lines): diff -r f147ddb83586 -r 613258f9f541 configure.ac --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configure.ac Mon Oct 29 22:42:34 2012 +0100 @@ -0,0 +1,140 @@ +AC_INIT([Pigeonhole], [0.4.0], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_CONFIG_AUX_DIR([.]) +AC_CONFIG_SRCDIR([src]) +AC_CONFIG_MACRO_DIR([m4]) + +# Autoheader is not needed and does more harm than good for this package. However, it is +# tightly integrated in autoconf/automake and therefore it is difficult not to use it. As +# a workaround we give autoheader a dummy config header to chew on and we handle the +# real config header ourselves. +AC_CONFIG_HEADERS([dummy-config.h pigeonhole-config.h]) + +AC_DEFINE_UNQUOTED(PIGEONHOLE_NAME, "$PACKAGE_NAME", + [Define to the full name of Pigeonhole for Dovecot.]) +AC_DEFINE_UNQUOTED(PIGEONHOLE_VERSION, "$PACKAGE_VERSION", + [Define to the version of Pigeonhole for Dovecot.]) + +AM_INIT_AUTOMAKE([no-define foreign tar-ustar]) + +AM_MAINTAINER_MODE + +AC_PROG_CC +AC_PROG_CPP +AC_PROG_LIBTOOL + +# Couple with Dovecot +# + +DC_DOVECOT +DC_DOVECOT_MODULEDIR +LIBDOVECOT_INCLUDE="$LIBDOVECOT_INCLUDE $LIBDOVECOT_STORAGE_INCLUDE" +CFLAGS="$DOVECOT_CFLAGS" +LIBS="$DOVECOT_LIBS" +AC_SUBST(LIBDOVECOT_INCLUDE) + +# Define Sieve documentation install dir +# + +sieve_docdir='${dovecot_docdir}/sieve' +AC_SUBST(sieve_docdir) + +# Extensions under development +# + +AC_ARG_WITH(unfinished-features, +[AC_HELP_STRING([--with-unfinished-features], + [Build unfinished new features/extensions [default=no]])], + if test x$withval = xno || test x$withval = xauto; then + want_unfinished_features=$withval + else + want_unfinished_features=yes + fi, + want_unfinished_features=no) +AM_CONDITIONAL(BUILD_UNFINISHED, test "$want_unfinished_features" = "yes") + +if test "$want_unfinished_features" = "yes"; then + AC_DEFINE(HAVE_SIEVE_UNFINISHED,, + [Define to build unfinished features/extensions.]) +fi + +# +# + +AC_ARG_WITH(docs, +[ --with-docs Install documentation (default)], + if test x$withval = xno; then + want_docs=no + else + want_docs=yes + fi, + want_docs=yes) +AM_CONDITIONAL(BUILD_DOCS, test "$want_docs" = "yes") + +AC_ARG_ENABLE(valgrind, +[AC_HELP_STRING([--enable-valgrind], [Enable Valgrind memory leak checks in testsuite [default=no]])], + if test x$enableval = xno || test x$enableval = xauto; then + want_valgrind=$enableval + else + want_valgrind=yes + fi, + want_valgrind=no) +AM_CONDITIONAL(TESTSUITE_VALGRIND, test "$want_valgrind" = "yes") + +AC_ARG_WITH(managesieve, +[AC_HELP_STRING([--with-managesieve], + [Build ManageSieve service [default=yes]])], + if test x$withval = xno || test x$withval = xauto; then + want_managesieve=$withval + else + want_managesieve=yes + fi, + want_managesieve=yes) +AM_CONDITIONAL(BUILD_MANAGESIEVE, test "$want_managesieve" = "yes") + +AC_CONFIG_FILES([ +Makefile +doc/Makefile +doc/man/Makefile +doc/example-config/Makefile +doc/example-config/conf.d/Makefile +doc/rfc/Makefile +src/Makefile +src/lib-sieve/Makefile +src/lib-sieve/plugins/Makefile +src/lib-sieve/plugins/vacation/Makefile +src/lib-sieve/plugins/subaddress/Makefile +src/lib-sieve/plugins/comparator-i-ascii-numeric/Makefile +src/lib-sieve/plugins/relational/Makefile +src/lib-sieve/plugins/regex/Makefile +src/lib-sieve/plugins/imap4flags/Makefile +src/lib-sieve/plugins/copy/Makefile +src/lib-sieve/plugins/include/Makefile +src/lib-sieve/plugins/body/Makefile +src/lib-sieve/plugins/variables/Makefile +src/lib-sieve/plugins/enotify/Makefile +src/lib-sieve/plugins/enotify/mailto/Makefile +src/lib-sieve/plugins/notify/Makefile +src/lib-sieve/plugins/environment/Makefile +src/lib-sieve/plugins/mailbox/Makefile +src/lib-sieve/plugins/date/Makefile +src/lib-sieve/plugins/spamvirustest/Makefile +src/lib-sieve/plugins/ihave/Makefile +src/lib-sieve/plugins/editheader/Makefile +src/lib-sieve/plugins/vnd.dovecot/Makefile +src/lib-sieve/plugins/vnd.dovecot/debug/Makefile +src/lib-sieve/plugins/vnd.dovecot/duplicate/Makefile +src/lib-sieve-tool/Makefile +src/lib-sievestorage/Makefile +src/lib-managesieve/Makefile +src/plugins/Makefile +src/plugins/lda-sieve/Makefile +src/sieve-tools/Makefile +src/managesieve/Makefile +src/managesieve-login/Makefile +src/testsuite/Makefile +stamp.h]) + +AC_OUTPUT + +echo +echo "NOTE: This is the UNSTABLE development branch of Pigeonhole for Dovecot v2.2." diff -r f147ddb83586 -r 613258f9f541 configure.in --- a/configure.in Wed Oct 17 22:22:02 2012 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,140 +0,0 @@ -AC_INIT([Pigeonhole], [0.4.0], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) -AC_CONFIG_AUX_DIR([.]) -AC_CONFIG_SRCDIR([src]) -AC_CONFIG_MACRO_DIR([m4]) - -# Autoheader is not needed and does more harm than good for this package. However, it is -# tightly integrated in autoconf/automake and therefore it is difficult not to use it. As -# a workaround we give autoheader a dummy config header to chew on and we handle the -# real config header ourselves. -AC_CONFIG_HEADERS([dummy-config.h pigeonhole-config.h]) - -AC_DEFINE_UNQUOTED(PIGEONHOLE_NAME, "$PACKAGE_NAME", - [Define to the full name of Pigeonhole for Dovecot.]) -AC_DEFINE_UNQUOTED(PIGEONHOLE_VERSION, "$PACKAGE_VERSION", - [Define to the version of Pigeonhole for Dovecot.]) - -AM_INIT_AUTOMAKE([no-define foreign tar-ustar]) - -AM_MAINTAINER_MODE - -AC_PROG_CC -AC_PROG_CPP -AC_PROG_LIBTOOL - -# Couple with Dovecot -# - -DC_DOVECOT -DC_DOVECOT_MODULEDIR -LIBDOVECOT_INCLUDE="$LIBDOVECOT_INCLUDE $LIBDOVECOT_STORAGE_INCLUDE" -CFLAGS="$DOVECOT_CFLAGS" -LIBS="$DOVECOT_LIBS" -AC_SUBST(LIBDOVECOT_INCLUDE) - -# Define Sieve documentation install dir -# - -sieve_docdir='${dovecot_docdir}/sieve' -AC_SUBST(sieve_docdir) - -# Extensions under development -# - -AC_ARG_WITH(unfinished-features, -[AC_HELP_STRING([--with-unfinished-features], - [Build unfinished new features/extensions [default=no]])], - if test x$withval = xno || test x$withval = xauto; then - want_unfinished_features=$withval - else - want_unfinished_features=yes - fi, - want_unfinished_features=no) -AM_CONDITIONAL(BUILD_UNFINISHED, test "$want_unfinished_features" = "yes") - -if test "$want_unfinished_features" = "yes"; then - AC_DEFINE(HAVE_SIEVE_UNFINISHED,, - [Define to build unfinished features/extensions.]) -fi - -# -# - -AC_ARG_WITH(docs, -[ --with-docs Install documentation (default)], - if test x$withval = xno; then - want_docs=no - else - want_docs=yes - fi, - want_docs=yes) -AM_CONDITIONAL(BUILD_DOCS, test "$want_docs" = "yes") - -AC_ARG_ENABLE(valgrind, -[AC_HELP_STRING([--enable-valgrind], [Enable Valgrind memory leak checks in testsuite [default=no]])], - if test x$enableval = xno || test x$enableval = xauto; then - want_valgrind=$enableval - else - want_valgrind=yes - fi, - want_valgrind=no) -AM_CONDITIONAL(TESTSUITE_VALGRIND, test "$want_valgrind" = "yes") - -AC_ARG_WITH(managesieve, -[AC_HELP_STRING([--with-managesieve], - [Build ManageSieve service [default=yes]])], - if test x$withval = xno || test x$withval = xauto; then - want_managesieve=$withval - else - want_managesieve=yes - fi, - want_managesieve=yes) -AM_CONDITIONAL(BUILD_MANAGESIEVE, test "$want_managesieve" = "yes") - -AC_CONFIG_FILES([ -Makefile -doc/Makefile -doc/man/Makefile -doc/example-config/Makefile -doc/example-config/conf.d/Makefile -doc/rfc/Makefile -src/Makefile -src/lib-sieve/Makefile -src/lib-sieve/plugins/Makefile -src/lib-sieve/plugins/vacation/Makefile -src/lib-sieve/plugins/subaddress/Makefile -src/lib-sieve/plugins/comparator-i-ascii-numeric/Makefile -src/lib-sieve/plugins/relational/Makefile -src/lib-sieve/plugins/regex/Makefile -src/lib-sieve/plugins/imap4flags/Makefile -src/lib-sieve/plugins/copy/Makefile -src/lib-sieve/plugins/include/Makefile -src/lib-sieve/plugins/body/Makefile -src/lib-sieve/plugins/variables/Makefile -src/lib-sieve/plugins/enotify/Makefile -src/lib-sieve/plugins/enotify/mailto/Makefile -src/lib-sieve/plugins/notify/Makefile -src/lib-sieve/plugins/environment/Makefile -src/lib-sieve/plugins/mailbox/Makefile -src/lib-sieve/plugins/date/Makefile -src/lib-sieve/plugins/spamvirustest/Makefile -src/lib-sieve/plugins/ihave/Makefile -src/lib-sieve/plugins/editheader/Makefile -src/lib-sieve/plugins/vnd.dovecot/Makefile -src/lib-sieve/plugins/vnd.dovecot/debug/Makefile -src/lib-sieve/plugins/vnd.dovecot/duplicate/Makefile -src/lib-sieve-tool/Makefile -src/lib-sievestorage/Makefile -src/lib-managesieve/Makefile -src/plugins/Makefile -src/plugins/lda-sieve/Makefile -src/sieve-tools/Makefile -src/managesieve/Makefile -src/managesieve-login/Makefile -src/testsuite/Makefile -stamp.h]) - -AC_OUTPUT - -echo -echo "NOTE: This is the UNSTABLE development branch of Pigeonhole for Dovecot v2.2." From dovecot at dovecot.org Tue Oct 30 15:06:05 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Oct 2012 15:06:05 +0200 Subject: dovecot-2.1: lib-lda: Avoid doing a costly mailbox sync when del... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/9cdeab12f3e1 changeset: 14792:9cdeab12f3e1 user: Timo Sirainen date: Tue Oct 30 15:05:45 2012 +0200 description: lib-lda: Avoid doing a costly mailbox sync when delivering mail to multiple recipients. diffstat: src/lib-lda/mail-deliver.c | 3 ++- src/lib-storage/index/index-sync.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletions(-) diffs (34 lines): diff -r 9d63f882194d -r 9cdeab12f3e1 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Mon Oct 29 19:04:36 2012 +0200 +++ b/src/lib-lda/mail-deliver.c Tue Oct 30 15:05:45 2012 +0200 @@ -331,7 +331,8 @@ ctx->saved_mail = TRUE; mail_deliver_log(ctx, "saved mail to %s", mailbox_name); - if (ctx->save_dest_mail && mailbox_sync(box, 0) == 0) { + if (ctx->save_dest_mail && + mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) == 0) { range = array_idx(&changes.saved_uids, 0); i_assert(range[0].seq1 == range[0].seq2); diff -r 9d63f882194d -r 9cdeab12f3e1 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Mon Oct 29 19:04:36 2012 +0200 +++ b/src/lib-storage/index/index-sync.c Tue Oct 30 15:05:45 2012 +0200 @@ -31,6 +31,17 @@ ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL) return FALSE; + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 && + (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* lib-lda is syncing the mailbox after saving a mail. + it only wants to find the new mail for potentially copying + to other mailboxes. that's mainly an optimization, and since + the mail was most likely already added to index we don't + need to do a full sync to find it. the main benefit here is + to avoid a very costly sync with a large Maildir/new/ */ + return FALSE; + } + if (ibox->notify_to != NULL) timeout_reset(ibox->notify_to); ibox->sync_last_check = ioloop_time; From dovecot at dovecot.org Wed Oct 31 13:11:18 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 13:11:18 +0200 Subject: dovecot-2.2: auth: Don't assert-crash in auth-token writing if p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6b45225ab1d5 changeset: 15361:6b45225ab1d5 user: Timo Sirainen date: Wed Oct 31 13:11:11 2012 +0200 description: auth: Don't assert-crash in auth-token writing if process isn't running as root. diffstat: src/auth/auth-token.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r d29a1bf4aba5 -r 6b45225ab1d5 src/auth/auth-token.c --- a/src/auth/auth-token.c Mon Oct 29 23:03:59 2012 +0200 +++ b/src/auth/auth-token.c Wed Oct 31 13:11:11 2012 +0200 @@ -105,8 +105,6 @@ mode_t old_mask; int fd, ret; - i_assert(getuid() == 0); - temp_path = t_strconcat(path, ".tmp", NULL); old_mask = umask(0); From dovecot at dovecot.org Wed Oct 31 13:12:42 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 13:12:42 +0200 Subject: dovecot-2.2: lib-dns: Added dns_lookup_switch_ioloop() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/285a88e00231 changeset: 15362:285a88e00231 user: Stephan Bosch date: Wed Oct 31 13:12:22 2012 +0200 description: lib-dns: Added dns_lookup_switch_ioloop() diffstat: src/lib-dns/dns-lookup.c | 7 +++++++ src/lib-dns/dns-lookup.h | 2 ++ 2 files changed, 9 insertions(+), 0 deletions(-) diffs (24 lines): diff -r 6b45225ab1d5 -r 285a88e00231 src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Wed Oct 31 13:11:11 2012 +0200 +++ b/src/lib-dns/dns-lookup.c Wed Oct 31 13:12:22 2012 +0200 @@ -190,3 +190,10 @@ { dns_lookup_free(lookup); } + +void dns_lookup_switch_ioloop(struct dns_lookup *lookup) +{ + if (lookup->to != NULL) + lookup->to = io_loop_move_timeout(&lookup->to); + lookup->io = io_loop_move_io(&lookup->io); +} diff -r 6b45225ab1d5 -r 285a88e00231 src/lib-dns/dns-lookup.h --- a/src/lib-dns/dns-lookup.h Wed Oct 31 13:11:11 2012 +0200 +++ b/src/lib-dns/dns-lookup.h Wed Oct 31 13:12:22 2012 +0200 @@ -39,4 +39,6 @@ /* Abort the DNS lookup without calling the callback. */ void dns_lookup_abort(struct dns_lookup **lookup); +void dns_lookup_switch_ioloop(struct dns_lookup *lookup); + #endif From dovecot at dovecot.org Wed Oct 31 13:24:59 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 13:24:59 +0200 Subject: dovecot-2.1: Makefile: Add -lssl to installed dovecot-config's L... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/7d931927e4ac changeset: 14793:7d931927e4ac user: Timo Sirainen date: Wed Oct 31 13:24:49 2012 +0200 description: Makefile: Add -lssl to installed dovecot-config's LIBDOVECOT_STORAGE if needed. diffstat: Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 9cdeab12f3e1 -r 7d931927e4ac Makefile.am --- a/Makefile.am Tue Oct 30 15:05:45 2012 +0200 +++ b/Makefile.am Wed Oct 31 13:24:49 2012 +0200 @@ -66,7 +66,7 @@ -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1=-ldovecot-login|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ - -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1=-ldovecot-storage|" \ + -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ -e "s|^\(LIBDOVECOT_INCLUDE\)=.*$$|\1=-I$(pkgincludedir)|" \ > $(DESTDIR)$(pkglibdir)/dovecot-config From dovecot at dovecot.org Wed Oct 31 13:29:23 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 13:29:23 +0200 Subject: dovecot-2.2: lib-storage: When mailbox_get_status/metadata() aut... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b690ff7a2d65 changeset: 15363:b690ff7a2d65 user: Timo Sirainen date: Wed Oct 31 13:29:16 2012 +0200 description: lib-storage: When mailbox_get_status/metadata() auto-syncs mailbox, use MAILBOX_SYNC_FLAG_FAST diffstat: src/lib-storage/index/index-status.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 285a88e00231 -r b690ff7a2d65 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Wed Oct 31 13:12:22 2012 +0200 +++ b/src/lib-storage/index/index-status.c Wed Oct 31 13:29:16 2012 +0200 @@ -38,7 +38,7 @@ return -1; } if (!box->synced) { - if (mailbox_sync(box, 0) < 0) + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) return -1; } index_storage_get_open_status(box, items, status_r); @@ -360,7 +360,7 @@ return -1; } if (!box->synced && (items & MAILBOX_METADATA_SYNC_ITEMS) != 0) { - if (mailbox_sync(box, 0) < 0) + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) return -1; } From dovecot at dovecot.org Wed Oct 31 13:34:58 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 13:34:58 +0200 Subject: dovecot-2.2: lib-storage: Don't require mailbox to be synced for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/53d4db36e251 changeset: 15364:53d4db36e251 user: Timo Sirainen date: Wed Oct 31 13:34:52 2012 +0200 description: lib-storage: Don't require mailbox to be synced for MAILBOX_METADATA_GUID lookup None of the mailbox formats currently require it, and lib-lda code assert-crashes with it since syncing is done with a transaction already open. diffstat: src/lib-storage/mail-storage.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b690ff7a2d65 -r 53d4db36e251 src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Wed Oct 31 13:29:16 2012 +0200 +++ b/src/lib-storage/mail-storage.h Wed Oct 31 13:34:52 2012 +0200 @@ -87,7 +87,7 @@ MAILBOX_METADATA_BACKEND_NAMESPACE = 0x10 /* metadata items that require mailbox to be synced at least once. */ #define MAILBOX_METADATA_SYNC_ITEMS \ - (MAILBOX_METADATA_GUID | MAILBOX_METADATA_VIRTUAL_SIZE) + (MAILBOX_METADATA_VIRTUAL_SIZE) }; enum mailbox_search_result_flags { From dovecot at dovecot.org Wed Oct 31 14:44:36 2012 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 31 Oct 2012 14:44:36 +0200 Subject: dovecot-2.2: hostpid_init(): Don't allow gethostname() to fail. ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/626a9df21e62 changeset: 15365:626a9df21e62 user: Timo Sirainen date: Wed Oct 31 14:44:24 2012 +0200 description: hostpid_init(): Don't allow gethostname() to fail. Be more strict about what chars it can contain. diffstat: src/lib/hostpid.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diffs (30 lines): diff -r 53d4db36e251 -r 626a9df21e62 src/lib/hostpid.c --- a/src/lib/hostpid.c Wed Oct 31 13:34:52 2012 +0200 +++ b/src/lib/hostpid.c Wed Oct 31 14:44:24 2012 +0200 @@ -6,6 +6,8 @@ #include #include +#define HOSTNAME_DISALLOWED_CHARS "/\r\n\t" + const char *my_hostname = NULL; const char *my_pid = NULL; @@ -15,14 +17,12 @@ { static char hostname[256], pid[MAX_INT_STRLEN]; - if (gethostname(hostname, sizeof(hostname)-1) == -1) { - if (i_strocpy(hostname, "unknown", sizeof(hostname)) < 0) - i_unreached(); - } + if (gethostname(hostname, sizeof(hostname)-1) == -1) + i_fatal("gethostname() failed: %m"); hostname[sizeof(hostname)-1] = '\0'; my_hostname = hostname; - if (strchr(hostname, '/') != NULL) + if (strcspn(hostname, HOSTNAME_DISALLOWED_CHARS) != strlen(hostname)) i_fatal("Invalid system hostname: %s", hostname); /* allow calling hostpid_init() multiple times to reset hostname */