dovecot-2.1: fts-solr: Truncate header if it's over 1 MB.
dovecot at dovecot.org
dovecot at dovecot.org
Fri Feb 22 14:49:38 EET 2013
details: http://hg.dovecot.org/dovecot-2.1/rev/0a932ba1f01f
changeset: 14920:0a932ba1f01f
user: Timo Sirainen <tss at iki.fi>
date: Fri Feb 22 14:49:27 2013 +0200
description:
fts-solr: Truncate header if it's over 1 MB.
Previously we just warned about it.
diffstat:
src/plugins/fts-solr/fts-backend-solr.c | 25 +++++++++++++++++--------
1 files changed, 17 insertions(+), 8 deletions(-)
diffs (75 lines):
diff -r 225c64ed0439 -r 0a932ba1f01f src/plugins/fts-solr/fts-backend-solr.c
--- a/src/plugins/fts-solr/fts-backend-solr.c Fri Feb 22 13:51:02 2013 +0200
+++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Feb 22 14:49:27 2013 +0200
@@ -17,9 +17,14 @@
#define SOLR_CMDBUF_SIZE (1024*64)
#define SOLR_CMDBUF_FLUSH_SIZE (SOLR_CMDBUF_SIZE-128)
-#define SOLR_BUFFER_WARN_SIZE (1024*1024)
#define SOLR_MAX_MULTI_ROWS 100000
+/* If header is larger than this, truncate it. */
+#define SOLR_HEADER_MAX_SIZE (1024*1024)
+/* If SOLR_HEADER_MAX_SIZE was already reached, write still to individual
+ header fields as long as they're smaller than this */
+#define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024
+
struct solr_fts_backend {
struct fts_backend backend;
};
@@ -42,12 +47,12 @@
ARRAY_DEFINE(fields, struct solr_fts_field);
uint32_t last_indexed_uid;
- uint32_t size_warned_uid;
unsigned int last_indexed_uid_set:1;
unsigned int body_open:1;
unsigned int documents_added:1;
unsigned int expunges:1;
+ unsigned int truncate_header:1;
};
static struct solr_connection *solr_conn = NULL;
@@ -445,6 +450,7 @@
fts_backend_solr_doc_close(ctx);
}
ctx->prev_uid = uid;
+ ctx->truncate_header = FALSE;
fts_backend_solr_doc_open(ctx, uid);
}
@@ -531,8 +537,11 @@
}
xml_encode_data(ctx->cmd, data, size);
} else {
- xml_encode_data(ctx->cur_value, data, size);
- if (ctx->cur_value2 != NULL)
+ if (!ctx->truncate_header)
+ xml_encode_data(ctx->cur_value, data, size);
+ if (ctx->cur_value2 != NULL &&
+ (!ctx->truncate_header ||
+ str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE))
xml_encode_data(ctx->cur_value2, data, size);
}
@@ -541,15 +550,15 @@
str_len(ctx->cmd));
str_truncate(ctx->cmd, 0);
}
- if (str_len(ctx->cur_value) >= SOLR_BUFFER_WARN_SIZE &&
- ctx->size_warned_uid != ctx->prev_uid) {
+ if (!ctx->truncate_header &&
+ str_len(ctx->cur_value) >= SOLR_HEADER_MAX_SIZE) {
/* a large header */
i_assert(ctx->cur_value != ctx->cmd);
- ctx->size_warned_uid = ctx->prev_uid;
- i_warning("fts-solr(%s): Mailbox %s UID=%u header size is huge",
+ i_warning("fts-solr(%s): Mailbox %s UID=%u header size is huge, truncating",
ctx->cur_box->storage->user->username,
mailbox_get_vname(ctx->cur_box), ctx->prev_uid);
+ ctx->truncate_header = TRUE;
}
return 0;
}
More information about the dovecot-cvs
mailing list