[dovecot-cvs] dovecot/src/plugins/fts-lucene fts-backend-lucene.c, 1.9, 1.10 lucene-wrapper.cc, 1.10, 1.11 lucene-wrapper.h, 1.6, 1.7
tss at dovecot.org
tss at dovecot.org
Sun Mar 25 19:21:13 EEST 2007
Update of /var/lib/cvs/dovecot/src/plugins/fts-lucene
In directory talvi:/tmp/cvs-serv2920
Modified Files:
fts-backend-lucene.c lucene-wrapper.cc lucene-wrapper.h
Log Message:
Create a locks directory under the lucene-indexed directory. Detect and
delete stale locks. Fixed expunges to work.
Index: fts-backend-lucene.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-lucene/fts-backend-lucene.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- fts-backend-lucene.c 15 Mar 2007 22:20:55 -0000 1.9
+++ fts-backend-lucene.c 25 Mar 2007 16:21:09 -0000 1.10
@@ -2,11 +2,13 @@
#include "lib.h"
#include "array.h"
+#include "mkdir-parents.h"
#include "mail-storage-private.h"
#include "lucene-wrapper.h"
#include "fts-lucene-plugin.h"
#define LUCENE_INDEX_DIR_NAME "lucene-indexes"
+#define LUCENE_LOCK_SUBDIR_NAME "locks"
struct lucene_mail_storage {
struct lucene_index *index;
@@ -35,7 +37,7 @@
{
struct lucene_mail_storage *lstorage;
struct lucene_fts_backend *backend;
- const char *path;
+ const char *path, *lock_path;
lstorage = LUCENE_CONTEXT(box->storage);
if (lstorage == NULL) {
@@ -47,9 +49,14 @@
}
path = t_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
+ lock_path = t_strdup_printf("%s/"LUCENE_LOCK_SUBDIR_NAME, path);
+ if (mkdir_parents(lock_path, 0700) < 0) {
+ i_error("mkdir_parents(%s) failed: %m", lock_path);
+ return NULL;
+ }
lstorage = i_new(struct lucene_mail_storage, 1);
- lstorage->index = lucene_index_init(path);
+ lstorage->index = lucene_index_init(path, lock_path);
array_idx_set(&box->storage->module_contexts,
fts_lucene_storage_module_id, &lstorage);
}
Index: lucene-wrapper.cc
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-lucene/lucene-wrapper.cc,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- lucene-wrapper.cc 15 Mar 2007 22:20:55 -0000 1.10
+++ lucene-wrapper.cc 25 Mar 2007 16:21:10 -0000 1.11
@@ -2,15 +2,28 @@
extern "C" {
#include "lib.h"
+#include "env-util.h"
#include "unichar.h"
+#include "str.h"
#include "str-sanitize.h"
#include "lucene-wrapper.h"
+
+#include <dirent.h>
+#include <sys/stat.h>
};
#include <CLucene.h>
/* Lucene's default is 10000. Use it here also.. */
#define MAX_TERMS_PER_DOCUMENT 10000
+/* If all the files in the lucene index directory are older than this many
+ seconds, assume we can delete stale locks */
+#define STALE_INDEX_SECS 60
+/* When index is determined to be stale, delete all locks older than this */
+#define STALE_LOCK_SECS 60
+/* Minimum interval between staleness checks */
+#define STALENESS_CHECK_INTERVAL 10
+
using namespace lucene::document;
using namespace lucene::index;
using namespace lucene::search;
@@ -18,10 +31,13 @@
using namespace lucene::analysis;
struct lucene_index {
- char *path;
+ char *path, *lock_path;
char *mailbox_name;
TCHAR *tmailbox_name;
+ time_t last_stale_check;
+ bool lock_error;
+
IndexReader *reader;
IndexWriter *writer;
IndexSearcher *searcher;
@@ -70,13 +86,96 @@
}
};
-struct lucene_index *lucene_index_init(const char *path)
+static bool lucene_dir_scan(const char *dir, const char *skip_path,
+ time_t stale_stamp, bool unlink_staled)
+{
+ DIR *d;
+ struct dirent *dp;
+ struct stat st;
+ string_t *path;
+ unsigned int dir_len;
+ bool found_nonstale = FALSE;
+
+ d = opendir(dir);
+ if (d == NULL) {
+ i_error("opendir(%s) failed: %m", dir);
+ return TRUE;
+ }
+
+ t_push();
+ path = t_str_new(256);
+ str_append(path, dir);
+ str_append_c(path, '/');
+ dir_len = str_len(path);
+
+ while ((dp = readdir(d)) != NULL) {
+ if (*dp->d_name == '.') {
+ if (dp->d_name[1] == '\0')
+ continue;
+ if (dp->d_name[1] == '.' && dp->d_name[2] == '\0')
+ continue;
+ }
+
+ str_truncate(path, dir_len);
+ str_append(path, dp->d_name);
+
+ if (skip_path != NULL &&
+ strcmp(str_c(path), skip_path) == 0)
+ continue;
+
+ if (stat(str_c(path), &st) < 0) {
+ if (errno != ENOENT)
+ i_error("stat(%s) failed: %m", str_c(path));
+ found_nonstale = TRUE;
+ } else if (st.st_ctime <= stale_stamp &&
+ st.st_mtime <= stale_stamp) {
+ if (unlink_staled) {
+ if (unlink(str_c(path)) < 0 &&
+ errno != ENOENT) {
+ i_error("unlink(%s) failed: %m",
+ str_c(path));
+ }
+ }
+ } else {
+ found_nonstale = TRUE;
+ }
+ }
+ if (closedir(d) < 0)
+ i_error("closedir(%s) failed: %m", dir);
+ t_pop();
+ return found_nonstale;
+}
+
+static void lucene_delete_stale_locks(struct lucene_index *index)
+{
+ time_t now;
+
+ now = time(NULL);
+ if (index->last_stale_check + STALENESS_CHECK_INTERVAL > now)
+ return;
+ index->last_stale_check = now;
+
+ if (lucene_dir_scan(index->path, index->lock_path,
+ now - STALE_INDEX_SECS, FALSE)) {
+ /* the index is probably being updated */
+ return;
+ }
+ (void)lucene_dir_scan(index->lock_path, NULL,
+ now - STALE_LOCK_SECS, TRUE);
+}
+
+struct lucene_index *lucene_index_init(const char *path, const char *lock_path)
{
struct lucene_index *index;
+ env_put(t_strconcat(LUCENE_LOCK_DIR_ENV_1"=", lock_path, NULL));
+
index = i_new(struct lucene_index, 1);
index->path = i_strdup(path);
+ index->lock_path = i_strdup(lock_path);
index->analyzer = _CLNEW DovecotAnalyzer();
+
+ lucene_delete_stale_locks(index);
return index;
}
@@ -93,6 +192,8 @@
_CLDELETE(index->analyzer);
i_free(index->mailbox_name);
i_free(index->tmailbox_name);
+ i_free(index->path);
+ i_free(index->lock_path);
i_free(index);
}
@@ -110,6 +211,23 @@
STRCPY_AtoT(index->tmailbox_name, mailbox_name, len);
}
+static void lucene_handle_error(struct lucene_index *index, CLuceneError &err,
+ const char *msg)
+{
+ const char *what = err.what();
+
+ if (err.number() == CL_ERR_IO && strncasecmp(what, "Lock", 4) == 0) {
+ /* "Lock obtain timed out". delete any stale locks. */
+ lucene_delete_stale_locks(index);
+ if (index->lock_error) {
+ /* we've already complained about this */
+ return;
+ }
+ index->lock_error = TRUE;
+ }
+ i_error("lucene index %s: %s failed: %s", index->path, msg, what);
+}
+
static int lucene_index_open(struct lucene_index *index)
{
if (index->reader != NULL)
@@ -121,7 +239,7 @@
try {
index->reader = IndexReader::open(index->path);
} catch (CLuceneError &err) {
- i_error("lucene: IndexReader::open(%s): %s", index->path, err.what());
+ lucene_handle_error(index, err, "IndexReader::open()");
return -1;
}
return 1;
@@ -213,9 +331,10 @@
deleted = true;
}
}
+ index->lock_error = FALSE;
_CLDELETE(hits);
} catch (CLuceneError &err) {
- i_error("lucene: last_uid search failed: %s", err.what());
+ lucene_handle_error(index, err, "last_uid search");
ret = -1;
}
@@ -244,15 +363,18 @@
i_assert(index->mailbox_name != NULL);
+ /* set this even if we fail so fts-storage won't crash */
+ *last_uid_r = index->last_uid;
+
lucene_index_close(index);
bool exists = IndexReader::indexExists(index->path);
try {
index->writer = _CLNEW IndexWriter(index->path,
index->analyzer, !exists);
+ index->lock_error = FALSE;
} catch (CLuceneError &err) {
- i_error("lucene: IndexWriter(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "IndexWriter()");
return -1;
}
index->writer->setMaxFieldLength(MAX_TERMS_PER_DOCUMENT);
@@ -273,8 +395,7 @@
try {
index->writer->addDocument(index->doc);
} catch (CLuceneError &err) {
- i_error("lucene: IndexWriter::addDocument(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "IndexWriter::addDocument()");
ret = -1;
}
@@ -335,8 +456,7 @@
index->writer->addDocument(&doc);
return 0;
} catch (CLuceneError &err) {
- i_error("lucene: IndexWriter::addDocument(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "IndexWriter::addDocument()");
return -1;
}
}
@@ -367,15 +487,13 @@
try {
index->writer->optimize();
} catch (CLuceneError &err) {
- i_error("lucene: IndexWriter::optimize(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "IndexWriter::optimize()");
ret = -1;
}
try {
index->writer->close();
} catch (CLuceneError &err) {
- i_error("lucene: IndexWriter::close(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "IndexWriter::close()");
ret = -1;
}
@@ -409,10 +527,18 @@
for (int32_t i = 0; i < hits->length(); i++)
index->reader->deleteDocument(hits->id(i));
+ index->lock_error = FALSE;
_CLDELETE(hits);
+ } catch (CLuceneError &err) {
+ lucene_handle_error(index, err, "expunge search");
+ return -1;
+ }
+
+ try {
+ index->reader->close();
return 0;
} catch (CLuceneError &err) {
- i_error("lucene: expunge search failed: %s", err.what());
+ lucene_handle_error(index, err, "IndexReader::close()");
return -1;
}
}
@@ -481,10 +607,10 @@
seq_range_array_add(result, 0, uid);
}
+ index->lock_error = FALSE;
_CLDELETE(hits);
} catch (CLuceneError &err) {
- i_error("lucene: search(%s) failed: %s",
- index->path, err.what());
+ lucene_handle_error(index, err, "search");
ret = -1;
}
Index: lucene-wrapper.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/plugins/fts-lucene/lucene-wrapper.h,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- lucene-wrapper.h 15 Mar 2007 22:20:55 -0000 1.6
+++ lucene-wrapper.h 25 Mar 2007 16:21:10 -0000 1.7
@@ -3,7 +3,7 @@
#include "fts-api-private.h"
-struct lucene_index *lucene_index_init(const char *path);
+struct lucene_index *lucene_index_init(const char *path, const char *lock_path);
void lucene_index_deinit(struct lucene_index *index);
int lucene_index_select_mailbox(struct lucene_index *index,
More information about the dovecot-cvs
mailing list