dovecot: When pipelining commands, delay synchronizations so tha...
dovecot at dovecot.org
dovecot at dovecot.org
Sat Jan 26 12:31:35 EET 2008
details: http://hg.dovecot.org/dovecot/rev/bca55675d77c
changeset: 7195:bca55675d77c
user: Timo Sirainen <tss at iki.fi>
date: Sat Jan 26 12:31:31 2008 +0200
description:
When pipelining commands, delay synchronizations so that only one combined
sync is performed for all the commands. This improves performance as well as
fixes some problems with the earlier method.
diffstat:
5 files changed, 213 insertions(+), 60 deletions(-)
src/imap/client.c | 36 ++++++--
src/imap/client.h | 10 ++
src/imap/cmd-append.c | 11 ++
src/imap/imap-sync.c | 215 +++++++++++++++++++++++++++++++++++++------------
src/imap/imap-sync.h | 1
diffs (truncated from 433 to 300 lines):
diff -r 226e80da0a23 -r bca55675d77c src/imap/client.c
--- a/src/imap/client.c Sat Jan 26 10:47:03 2008 +0200
+++ b/src/imap/client.c Sat Jan 26 12:31:31 2008 +0200
@@ -49,7 +49,7 @@ struct client *client_create(int fd_in,
client->to_idle = timeout_add(CLIENT_IDLE_TIMEOUT_MSECS,
client_idle_timeout, client);
- client->command_pool = pool_alloconly_create("client command", 8192);
+ client->command_pool = pool_alloconly_create("client command", 1024*12);
client->namespaces = namespaces;
while (namespaces != NULL) {
@@ -512,8 +512,18 @@ static void client_idle_output_timeout(s
bool client_handle_unfinished_cmd(struct client_command_context *cmd)
{
- if (cmd->state != CLIENT_COMMAND_STATE_WAIT_OUTPUT)
- return FALSE;
+ if (cmd->state == CLIENT_COMMAND_STATE_WAIT_INPUT) {
+ /* need more input */
+ return FALSE;
+ }
+ if (cmd->state != CLIENT_COMMAND_STATE_WAIT_OUTPUT) {
+ /* waiting for something */
+ if (cmd->state == CLIENT_COMMAND_STATE_WAIT_SYNC) {
+ /* this is mainly for APPEND. */
+ client_add_missing_io(cmd->client);
+ }
+ return TRUE;
+ }
/* output is blocking, we can execute more commands */
o_stream_set_flush_pending(cmd->client->output, TRUE);
@@ -586,7 +596,7 @@ static bool client_command_input(struct
}
}
-static int client_handle_next_command(struct client *client, bool *remove_io_r)
+static bool client_handle_next_command(struct client *client, bool *remove_io_r)
{
size_t size;
@@ -630,7 +640,6 @@ static bool client_handle_input(struct c
{
bool ret, remove_io, handled_commands = FALSE;
- o_stream_cork(client->output);
client->handling_input = TRUE;
do {
T_FRAME(
@@ -638,9 +647,8 @@ static bool client_handle_input(struct c
);
if (ret)
handled_commands = TRUE;
- } while (ret && !client->disconnected);
+ } while (ret && !client->disconnected && client->io != NULL);
client->handling_input = FALSE;
- o_stream_uncork(client->output);
if (client->output->closed) {
client_destroy(client, NULL);
@@ -650,13 +658,20 @@ static bool client_handle_input(struct c
io_remove(&client->io);
else
client_add_missing_io(client);
- return handled_commands;
+ if (!handled_commands)
+ return FALSE;
+
+ ret = cmd_sync_delayed(client);
+ if (ret)
+ client_continue_pending_input(&client);
+ return TRUE;
}
}
void client_input(struct client *client)
{
struct client_command_context *cmd;
+ struct ostream *output = client->output;
ssize_t bytes;
i_assert(client->io != NULL);
@@ -671,6 +686,8 @@ void client_input(struct client *client)
return;
}
+ o_stream_ref(output);
+ o_stream_cork(output);
if (!client_handle_input(client) && bytes == -2) {
/* parameter word is longer than max. input buffer size.
this is most likely an error, so skip the new data
@@ -683,6 +700,8 @@ void client_input(struct client *client)
client_send_command_error(cmd, "Too long argument.");
client_command_free(cmd);
}
+ o_stream_uncork(output);
+ o_stream_unref(&output);
}
static void client_output_cmd(struct client_command_context *cmd)
@@ -750,6 +769,7 @@ int client_output(struct client *client)
client_destroy(client, NULL);
return 1;
} else {
+ (void)cmd_sync_delayed(client);
client_continue_pending_input(&client);
}
return ret;
diff -r 226e80da0a23 -r bca55675d77c src/imap/client.h
--- a/src/imap/client.h Sat Jan 26 10:47:03 2008 +0200
+++ b/src/imap/client.h Sat Jan 26 12:31:31 2008 +0200
@@ -27,6 +27,8 @@ enum client_command_state {
CLIENT_COMMAND_STATE_WAIT_OUTPUT,
/* Wait for other commands to finish execution */
CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY,
+ /* Waiting for other commands to finish so we can sync */
+ CLIENT_COMMAND_STATE_WAIT_SYNC,
/* Command is finished */
CLIENT_COMMAND_STATE_DONE
};
@@ -46,6 +48,13 @@ struct client_command_context {
struct imap_parser *parser;
enum client_command_state state;
+ /* if multiple commands are in progress, we may need to wait for them
+ to finish before syncing mailbox. */
+ unsigned int sync_counter;
+ enum mailbox_sync_flags sync_flags;
+ enum imap_sync_flags sync_imap_flags;
+ const char *sync_tagline;
+
unsigned int uid:1; /* used UID command */
unsigned int cancel:1; /* command is wanted to be cancelled */
unsigned int param_error:1;
@@ -63,6 +72,7 @@ struct client {
struct mailbox *mailbox;
struct mailbox_keywords keywords;
unsigned int select_counter; /* increased when mailbox is changed */
+ unsigned int sync_counter;
uint32_t messages_count, recent_count, uidvalidity;
time_t last_input, last_output;
diff -r 226e80da0a23 -r bca55675d77c src/imap/cmd-append.c
--- a/src/imap/cmd-append.c Sat Jan 26 10:47:03 2008 +0200
+++ b/src/imap/cmd-append.c Sat Jan 26 12:31:31 2008 +0200
@@ -78,10 +78,10 @@ static void client_input_append(struct c
o_stream_uncork(client->output);
if (!finished && cmd->state != CLIENT_COMMAND_STATE_DONE)
(void)client_handle_unfinished_cmd(cmd);
- else {
+ else
client_command_free(cmd);
+ if (cmd_sync_delayed(client))
client_continue_pending_input(&client);
- }
}
/* Returns -1 = error, 0 = need more data, 1 = successful. flags and
@@ -455,6 +455,13 @@ bool cmd_append(struct client_command_co
struct cmd_append_context *ctx;
const char *mailbox;
+ if (client->syncing) {
+ /* if transaction is created while its view is synced,
+ appends aren't allowed for it. */
+ cmd->state = CLIENT_COMMAND_STATE_WAIT_UNAMBIGUITY;
+ return FALSE;
+ }
+
/* <mailbox> */
if (!client_read_string_args(cmd, 1, &mailbox))
return FALSE;
diff -r 226e80da0a23 -r bca55675d77c src/imap/imap-sync.c
--- a/src/imap/imap-sync.c Sat Jan 26 10:47:03 2008 +0200
+++ b/src/imap/imap-sync.c Sat Jan 26 12:31:31 2008 +0200
@@ -2,15 +2,11 @@
#include "common.h"
#include "str.h"
+#include "ostream.h"
#include "mail-storage.h"
#include "imap-util.h"
#include "imap-sync.h"
#include "commands.h"
-
-struct cmd_sync_context {
- const char *tagline;
- struct imap_sync_context *sync_ctx;
-};
struct imap_sync_context {
struct client *client;
@@ -189,46 +185,82 @@ int imap_sync_more(struct imap_sync_cont
return ret;
}
-static bool cmd_sync_continue(struct client_command_context *cmd)
-{
- struct cmd_sync_context *ctx = cmd->context;
+static bool cmd_sync_continue(struct client_command_context *sync_cmd)
+{
+ struct client_command_context *cmd;
+ struct client *client = sync_cmd->client;
+ struct imap_sync_context *ctx = sync_cmd->context;
int ret;
- if (cmd->cancel)
- ret = 0;
- else {
- if ((ret = imap_sync_more(ctx->sync_ctx)) == 0)
- return FALSE;
- }
-
+ i_assert(ctx->client == client);
+
+ if ((ret = imap_sync_more(ctx)) == 0)
+ return FALSE;
if (ret < 0)
- ctx->sync_ctx->failed = TRUE;
-
- cmd->client->syncing = FALSE;
- if (imap_sync_deinit(ctx->sync_ctx) < 0) {
- client_send_untagged_storage_error(cmd->client,
- mailbox_get_storage(cmd->client->mailbox));
- }
-
- if (!cmd->cancel)
- client_send_tagline(cmd, ctx->tagline);
+ ctx->failed = TRUE;
+
+ client->syncing = FALSE;
+ if (imap_sync_deinit(ctx) < 0) {
+ client_send_untagged_storage_error(client,
+ mailbox_get_storage(client->mailbox));
+ }
+ sync_cmd->context = NULL;
+
+ /* finish all commands that waited for this sync */
+ for (cmd = client->command_queue; cmd != NULL; cmd = cmd->next) {
+ if (cmd->state == CLIENT_COMMAND_STATE_WAIT_SYNC &&
+ cmd != sync_cmd &&
+ cmd->sync_counter+1 == client->sync_counter) {
+ client_send_tagline(cmd, cmd->sync_tagline);
+ client_command_free(cmd);
+ }
+ }
+ client_send_tagline(sync_cmd, sync_cmd->sync_tagline);
return TRUE;
}
-bool cmd_sync(struct client_command_context *cmd, enum mailbox_sync_flags flags,
- enum imap_sync_flags imap_flags, const char *tagline)
-{
- struct client *client = cmd->client;
- struct cmd_sync_context *ctx;
+static void get_common_sync_flags(struct client *client,
+ enum mailbox_sync_flags *flags_r,
+ enum imap_sync_flags *imap_flags_r)
+{
+ struct client_command_context *cmd;
+ unsigned int count = 0, fast_count = 0, noexpunges_count = 0;
+
+ *flags_r = 0;
+ *imap_flags_r = 0;
+
+ for (cmd = client->command_queue; cmd != NULL; cmd = cmd->next) {
+ if (cmd->sync_tagline != NULL &&
+ cmd->sync_counter == client->sync_counter) {
+ if ((cmd->sync_flags & MAILBOX_SYNC_FLAG_FAST) != 0)
+ fast_count++;
+ if (cmd->sync_flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES)
+ noexpunges_count++;
+ *flags_r |= cmd->sync_flags;
+ *imap_flags_r |= cmd->sync_imap_flags;
+ count++;
+ }
+ }
+ if (fast_count != count)
+ *flags_r &= ~MAILBOX_SYNC_FLAG_FAST;
+ if (noexpunges_count != count)
+ *flags_r &= ~MAILBOX_SYNC_FLAG_NO_EXPUNGES;
+
+ i_assert((*flags_r & (MAILBOX_SYNC_AUTO_STOP |
+ MAILBOX_SYNC_FLAG_FIX_INCONSISTENT)) == 0);
+}
+
+static bool cmd_sync_client(struct client_command_context *sync_cmd)
+{
+ struct client *client = sync_cmd->client;
+ struct imap_sync_context *ctx;
+ enum mailbox_sync_flags flags;
+ enum imap_sync_flags imap_flags;
bool no_newmail;
- i_assert(client->output_lock == cmd || client->output_lock == NULL);
-
- if (client->mailbox == NULL ||
- mailbox_transaction_get_count(client->mailbox) > 0) {
- client_send_tagline(cmd, tagline);
More information about the dovecot-cvs
mailing list