dovecot-2.2: lib-fts: Removed tokenizer name macros from fts-tok...
dovecot at dovecot.org
dovecot at dovecot.org
Sat May 9 12:00:27 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/12aeb1ac8f0d
changeset: 18590:12aeb1ac8f0d
user: Timo Sirainen <tss at iki.fi>
date: Sat May 09 14:56:33 2015 +0300
description:
lib-fts: Removed tokenizer name macros from fts-tokenizer.h
The tokenizers can be directly accessed via their class structs.
diffstat:
src/lib-fts/fts-tokenizer-address.c | 2 +-
src/lib-fts/fts-tokenizer-generic.c | 2 +-
src/lib-fts/fts-tokenizer.h | 2 --
src/lib-fts/test-fts-tokenizer.c | 29 ++++++++++++++---------------
4 files changed, 16 insertions(+), 19 deletions(-)
diffs (153 lines):
diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer-address.c
--- a/src/lib-fts/fts-tokenizer-address.c Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-address.c Sat May 09 14:56:33 2015 +0300
@@ -305,7 +305,7 @@
};
static const struct fts_tokenizer fts_tokenizer_email_address_real = {
- .name = FTS_TOKENIZER_EMAIL_ADDRESS_NAME,
+ .name = "email-address",
.v = &email_address_tokenizer_vfuncs
};
const struct fts_tokenizer *fts_tokenizer_email_address =
diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer-generic.c
--- a/src/lib-fts/fts-tokenizer-generic.c Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer-generic.c Sat May 09 14:56:33 2015 +0300
@@ -596,7 +596,7 @@
};
static const struct fts_tokenizer fts_tokenizer_generic_real = {
- .name = FTS_TOKENIZER_GENERIC_NAME,
+ .name = "generic",
.v = &generic_tokenizer_vfuncs
};
const struct fts_tokenizer *fts_tokenizer_generic = &fts_tokenizer_generic_real;
diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/fts-tokenizer.h
--- a/src/lib-fts/fts-tokenizer.h Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/fts-tokenizer.h Sat May 09 14:56:33 2015 +0300
@@ -23,7 +23,6 @@
further. Defaults to disabled. Enable by defining the keyword (and any
value). */
extern const struct fts_tokenizer *fts_tokenizer_email_address;
-#define FTS_TOKENIZER_EMAIL_ADDRESS_NAME "email-address"
/* Generic email content tokenizer. Cuts text into tokens. */
/* Settings:
@@ -41,7 +40,6 @@
differ in some details, e.g. simple will cut "a.b" and tr29 will
not. The default is "simple" */
extern const struct fts_tokenizer *fts_tokenizer_generic;
-#define FTS_TOKENIZER_GENERIC_NAME "generic"
/*
Tokenizing workflow, find --> create --> filter --> destroy.
diff -r 5937f4d505c5 -r 12aeb1ac8f0d src/lib-fts/test-fts-tokenizer.c
--- a/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:55:47 2015 +0300
+++ b/src/lib-fts/test-fts-tokenizer.c Sat May 09 14:56:33 2015 +0300
@@ -11,6 +11,14 @@
#include <stdlib.h>
+static void test_fts_tokenizer_find(void)
+{
+ test_begin("fts tokenizer find");
+ test_assert(fts_tokenizer_find("email-address") == fts_tokenizer_email_address);
+ test_assert(fts_tokenizer_find("generic") == fts_tokenizer_generic);
+ test_end();
+}
+
static void test_fts_tokenizer_generic_only(void)
{
static const unsigned char input[] =
@@ -22,14 +30,12 @@
"and", "longlonglongabcdefghijklmnopqr",
"more", "Hello", "world", "last", NULL
};
- const struct fts_tokenizer *tok_class;
struct fts_tokenizer *tok;
const char * const *eopp = expected_output;
const char *token, *error;
test_begin("fts tokenizer generic simple");
- tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
- test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);
+ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0);
/*TODO: Uncomment when fts-tokenizer-generic-private.h inclusion is fixed */
/*test_assert(((struct generic_fts_tokenizer *) tok)->algorithm == BOUNDARY_ALGORITHM_SIMPLE);*/
while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
@@ -57,15 +63,13 @@
"there", "was", "text", "galore",
"and", "more", NULL
};
- const struct fts_tokenizer *tok_class;
struct fts_tokenizer *tok;
const char * const *eopp = expected_output;
const char *token, *error;
test_begin("fts tokenizer generic simple with Unicode whitespace");
fts_tokenizer_register(fts_tokenizer_generic);
- tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
- test_assert(fts_tokenizer_create(tok_class, NULL, NULL, &tok, &error) == 0);
+ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &tok, &error) == 0);
while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
test_assert(strcmp(token, *eopp) == 0);
eopp++;
@@ -132,15 +136,13 @@
"and", "more", "Hello", "world", "3.14",
"3,14", "last", "longlonglongabcdefghijklmnopqr", "1", NULL
};
- const struct fts_tokenizer *tok_class;
struct fts_tokenizer *tok;
const char * const *eopp = expected_output;
const char *token, *error;
test_begin("fts tokenizer generic TR29");
fts_tokenizer_register(fts_tokenizer_generic);
- tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
- test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
test_assert(strcmp(token, *eopp) == 0);
eopp++;
@@ -169,15 +171,13 @@
"there", "was", "text", "galore",
"and", "more", NULL
};
- const struct fts_tokenizer *tok_class;
struct fts_tokenizer *tok;
const char * const *eopp = expected_output;
const char *token, *error;
test_begin("fts tokenizer generic TR29 with Unicode whitespace");
fts_tokenizer_register(fts_tokenizer_generic);
- tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
- test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
test_assert(strcmp(token, *eopp) == 0);
eopp++;
@@ -200,15 +200,13 @@
static const char *const expected_output[] = {
"hello", "world", NULL
};
- const struct fts_tokenizer *tok_class;
struct fts_tokenizer *tok;
const char * const *eopp = expected_output;
const char *token, *error;
test_begin("fts tokenizer generic TR29 with MinNumLet U+FF0E at end");
fts_tokenizer_register(fts_tokenizer_generic);
- tok_class = fts_tokenizer_find(FTS_TOKENIZER_GENERIC_NAME);
- test_assert(fts_tokenizer_create(tok_class, NULL, tr29_settings, &tok, &error) == 0);
+ test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, tr29_settings, &tok, &error) == 0);
while (fts_tokenizer_next(tok, input, sizeof(input)-1, &token) > 0) {
test_assert(null_strcmp(token, *eopp) == 0);
eopp++;
@@ -581,6 +579,7 @@
int main(void)
{
static void (*test_functions[])(void) = {
+ test_fts_tokenizer_find,
test_fts_tokenizer_generic_only,
test_fts_tokenizer_generic_unicode_whitespace,
test_fts_tokenizer_char_generic_only,
More information about the dovecot-cvs
mailing list