/// <summary>
/// Runs the indexing process.
/// </summary>
/// <param name="users">The users.</param>
/// <param name="created">if set to <c>true</c> [created].</param>
public void CreateOrUpdate(IList<User> users, bool created)
{
var modifier = new IndexModifier(
Setting.UserSearchIndexPath.Value,
new StandardAnalyzer(),
(!created) ? true : false
);
foreach (var user in users)
{
if (created)
{
modifier.DeleteDocuments(new Term("id", user.Id.ToString()));
}
var document = new Document();
UserToDocument(user, document);
modifier.AddDocument(document);
}
modifier.Optimize();
modifier.Close();
}
/// <summary>
/// Deletes the index.
/// </summary>
/// <param name="indexPath">The index path.</param>
/// <param name="docList">The doc list.</param>
/// <returns></returns>
public bool DeleteIndex(string indexPath, List <string> docList)
{
bool result = false;
try
{
using (Lucene.Net.Store.Directory directory = FSDirectory.Open(new DirectoryInfo(indexPath)))
{
Analyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29);
var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, RegularExpression, analyzer);
IndexModifier modifier = new Lucene.Net.Index.IndexModifier(indexPath, analyzer, false);
foreach (string doc in docList)
{
modifier.DeleteDocuments(new Lucene.Net.Index.Term(FilePath, doc));
}
modifier.Flush();
modifier.Close();
}
}
catch (Exception ex)
{
// Swallow Exceptions here
}
return(result);
}
internal IndexThread(IndexModifier index, int maxWait, int threadNumber)
{
this.index = index;
this.maxWait = maxWait;
this.threadNumber = threadNumber;
// TODO: test case is not reproducible despite pseudo-random numbers:
random = new System.Random((System.Int32)(101 + threadNumber)); // constant seed for better reproducability
}
private void TestIndexInternal(int maxWait)
{
bool create = true;
//Directory rd = new RAMDirectory();
// work on disk to make sure potential lock problems are tested:
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
{
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
}
System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex"));
Directory rd = FSDirectory.Open(indexDir);
IndexThread.id = 0;
IndexThread.idStack.Clear();
IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create);
IndexThread thread1 = new IndexThread(index, maxWait, 1);
thread1.Start();
IndexThread thread2 = new IndexThread(index, maxWait, 2);
thread2.Start();
while (thread1.IsAlive || thread2.IsAlive)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100));
}
index.Optimize();
int added = thread1.added + thread2.added;
int deleted = thread1.deleted + thread2.deleted;
Assert.AreEqual(added - deleted, index.DocCount());
index.Close();
try
{
index.Close();
Assert.Fail();
}
catch (System.SystemException e)
{
// expected exception
}
RmDir(indexDir);
}
private void ConstructModifier()
{
if (_directory == null) throw new InvalidOperationException("Directory is not specified");
using (EnterWriteLockWithoutSearcherRefresh()) {
try {
_modifier = new IndexModifier(_directory, _defaultAnalyzer ?? new SimpleAnalyzer(), !DoesIndexExist());
} catch (Exception ex) {
var _writer = new IndexWriter(_directory, _defaultAnalyzer ?? new SimpleAnalyzer(), true);
_writer.Close();
_modifier = new IndexModifier(_directory, _defaultAnalyzer ?? new SimpleAnalyzer(), false);
}
}
}
private void CloseModifier()
{
using (EnterWriteLockWithoutSearcherRefresh()) {
if (_modifier != null) {
_modifier.Close();
_modifier = null;
}
}
}
public void threadproc_update(object obj)
{
lock (locker) // If a thread is updating the index, no other thread should be doing anything with it.
{
try
{
if (searcher != null)
{
try
{
searcher.Close();
}
catch (Exception e)
{
}
searcher = null;
}
Lucene.Net.Index.IndexModifier modifier = new Lucene.Net.Index.IndexModifier(DBNLConfigurationManager.LuceneElement.IndexingFolder, analyzer, false);
// same as build, but uses "modifier" instead of write.
// uses additional "where" clause for bugid
int id = (int)obj;
modifier.DeleteDocuments(new Lucene.Net.Index.Term("id", Convert.ToString(id)));
var item = new ContentService().GetContentById(id);
modifier.AddDocument(create_doc(
item.ContentId, item.Content1));
modifier.Flush();
modifier.Close();
}
catch (Exception e)
{
}
}
}
public virtual void TestIndex()
{
Directory ramDir = new RAMDirectory();
IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true);
i.AddDocument(GetDoc());
Assert.AreEqual(1, i.DocCount());
i.Flush();
i.AddDocument(GetDoc(), new SimpleAnalyzer());
Assert.AreEqual(2, i.DocCount());
i.Optimize();
Assert.AreEqual(2, i.DocCount());
i.Flush();
i.DeleteDocument(0);
Assert.AreEqual(1, i.DocCount());
i.Flush();
Assert.AreEqual(1, i.DocCount());
i.AddDocument(GetDoc());
i.AddDocument(GetDoc());
i.Flush();
// depend on merge policy - Assert.AreEqual(3, i.docCount());
i.DeleteDocuments(allDocTerm);
Assert.AreEqual(0, i.DocCount());
i.Optimize();
Assert.AreEqual(0, i.DocCount());
// Lucene defaults:
Assert.IsNull(i.GetInfoStream());
Assert.IsTrue(i.GetUseCompoundFile());
Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs());
Assert.AreEqual(10000, i.GetMaxFieldLength());
Assert.AreEqual(10, i.GetMergeFactor());
// test setting properties:
i.SetMaxBufferedDocs(100);
i.SetMergeFactor(25);
i.SetMaxFieldLength(250000);
i.AddDocument(GetDoc());
i.SetUseCompoundFile(false);
i.Flush();
Assert.AreEqual(100, i.GetMaxBufferedDocs());
Assert.AreEqual(25, i.GetMergeFactor());
Assert.AreEqual(250000, i.GetMaxFieldLength());
Assert.IsFalse(i.GetUseCompoundFile());
// test setting properties when internally the reader is opened:
i.DeleteDocuments(allDocTerm);
i.SetMaxBufferedDocs(100);
i.SetMergeFactor(25);
i.SetMaxFieldLength(250000);
i.AddDocument(GetDoc());
i.SetUseCompoundFile(false);
i.Optimize();
Assert.AreEqual(100, i.GetMaxBufferedDocs());
Assert.AreEqual(25, i.GetMergeFactor());
Assert.AreEqual(250000, i.GetMaxFieldLength());
Assert.IsFalse(i.GetUseCompoundFile());
i.Close();
try
{
i.DocCount();
Assert.Fail();
}
catch (System.SystemException e)
{
// expected exception
}
}
internal IndexThread(IndexModifier index, int maxWait, int threadNumber)
{
this.index = index;
this.maxWait = maxWait;
this.threadNumber = threadNumber;
// TODO: test case is not reproducible despite pseudo-random numbers:
random = new System.Random((System.Int32) (101 + threadNumber)); // constant seed for better reproducability
}
public virtual void TestIndex()
{
Directory ramDir = new RAMDirectory();
IndexModifier i = new IndexModifier(ramDir, new StandardAnalyzer(), true);
i.AddDocument(GetDoc());
Assert.AreEqual(1, i.DocCount());
i.Flush();
i.AddDocument(GetDoc(), new SimpleAnalyzer());
Assert.AreEqual(2, i.DocCount());
i.Optimize();
Assert.AreEqual(2, i.DocCount());
i.Flush();
i.DeleteDocument(0);
Assert.AreEqual(1, i.DocCount());
i.Flush();
Assert.AreEqual(1, i.DocCount());
i.AddDocument(GetDoc());
i.AddDocument(GetDoc());
i.Flush();
// depend on merge policy - Assert.AreEqual(3, i.docCount());
i.DeleteDocuments(allDocTerm);
Assert.AreEqual(0, i.DocCount());
i.Optimize();
Assert.AreEqual(0, i.DocCount());
// Lucene defaults:
Assert.IsNull(i.GetInfoStream());
Assert.IsTrue(i.GetUseCompoundFile());
Assert.AreEqual(IndexWriter.DISABLE_AUTO_FLUSH, i.GetMaxBufferedDocs());
Assert.AreEqual(10000, i.GetMaxFieldLength());
Assert.AreEqual(10, i.GetMergeFactor());
// test setting properties:
i.SetMaxBufferedDocs(100);
i.SetMergeFactor(25);
i.SetMaxFieldLength(250000);
i.AddDocument(GetDoc());
i.SetUseCompoundFile(false);
i.Flush();
Assert.AreEqual(100, i.GetMaxBufferedDocs());
Assert.AreEqual(25, i.GetMergeFactor());
Assert.AreEqual(250000, i.GetMaxFieldLength());
Assert.IsFalse(i.GetUseCompoundFile());
// test setting properties when internally the reader is opened:
i.DeleteDocuments(allDocTerm);
i.SetMaxBufferedDocs(100);
i.SetMergeFactor(25);
i.SetMaxFieldLength(250000);
i.AddDocument(GetDoc());
i.SetUseCompoundFile(false);
i.Optimize();
Assert.AreEqual(100, i.GetMaxBufferedDocs());
Assert.AreEqual(25, i.GetMergeFactor());
Assert.AreEqual(250000, i.GetMaxFieldLength());
Assert.IsFalse(i.GetUseCompoundFile());
i.Close();
try
{
i.DocCount();
Assert.Fail();
}
catch (System.SystemException e)
{
// expected exception
}
}
/// <summary>
/// Creates/Updates the index for a given host. If the index already exists then we'll update it other
/// wise we create a new index for the host. Each host index is store in its own folder off the base directory.
/// </summary>
/// <param name="hostId"></param>
private void GenerateIndex(int hostId, DateTime lastUpdateTime)
{
Log.InfoFormat("Starting index generation for HostID: {0}", hostId);
IndexModifier modifier = null;
bool isIncrementalCrawl = false;
int storiesToIndexCount = 0;
int pageSize;
try
{
bool indexExists = IndexExists(hostId);
if (indexExists)
{
//logic to version the lucene index to allow
//use to replace with a new version should we need to
//this should be kind of value stored in settings table.
//check if the version of the lucene index is the latest version
indexExists = IsLuceneIndexCorrectVersion(hostId);
if (!indexExists)
{
Log.Debug("Lucene index exists but is older version, need to overwrite with new document fields");
}
}
StoryCollection stories = null;
if (!indexExists)
{
//the index doesnt exist so we are going to do a full index of
//all stories in the database
Log.InfoFormat("Creating a new index HostID: {0}", hostId);
isIncrementalCrawl = false;
storiesToIndexCount = Story.GetAllStoriesCount(hostId);
}
else
{
if (!HostCrawlSuccessful(hostId))
{
//force the last update time to a low value to get all the records
//since we need to recrawl the index fully
lastUpdateTime = DateTime.Parse("1/1/1975");
Log.InfoFormat("Last crawl didnt complete successfully, attempting a full crawl");
}
else
Log.InfoFormat("Updating existing index");
isIncrementalCrawl = true;
storiesToIndexCount = Story.GetUpdatedStoriesCount(hostId, lastUpdateTime);
Log.InfoFormat("Found: {0} stories to add to index since last update at: {1}",
storiesToIndexCount,
lastUpdateTime);
}
if (storiesToIndexCount == 0)
{
Log.InfoFormat("Nothing todo, no new stories to crawl for HostID: {0}", hostId);
isUpdateRunning = false;
return;
}
modifier = new IndexModifier(IndexHostPath(hostId), new DnkAnalyzer(), !indexExists);
modifier.SetMaxBufferedDocs(50);
modifier.SetMergeFactor(150);
SearchSettings searchSettings = new SearchSettings();
pageSize = searchSettings.StoriesPageSize;
int pageTotal = CalculateNumberOfPages(storiesToIndexCount, pageSize);
for (int i = 1; i <= pageTotal; i++)
{
if (isIncrementalCrawl)
stories = Story.GetUpdatedStories(hostId, lastUpdateTime, i, pageSize);
else
stories = Story.GetAllStories(hostId, i, pageSize);
AddStoriesToIndex(modifier, isIncrementalCrawl, stories);
}
modifier.Optimize();
Log.InfoFormat("index optimized for HostID:{0}", hostId);
modifier.Close();
Log.InfoFormat("Index Modifier closed for Host:{0}", hostId);
modifier = null;
//we completed ok
RecordHostCrawlSuccess(hostId, true);
}
catch (Exception ex)
{
RecordHostCrawlSuccess(hostId, false);
Log.ErrorFormat("Error occurred while adding items to the index HostID:{0}, message: {1}",hostId, ex.Message);
}
finally
{
//attempt to close the modifier if it still exists
if (modifier != null)
{
try
{
modifier.Close();
modifier = null;
Log.InfoFormat("Able to close the Index Modifier in the final block, HostID:{0}", hostId);
}
catch (Exception ex)
{
Log.ErrorFormat("Unable to close the modifer in the final block HostID:{0} Message:{1}", hostId, ex.Message);
}
}
}
}
/// <summary>
/// Loops thro a list of stories and adds them to the index. If the crawl is an incremental
/// update then first the story is removed then added again.
/// </summary>
/// <param name="modifier">IndexModifer used to update the index</param>
/// <param name="isIncrementalCrawl">bool indicating if the stories should
/// be removed from the existing index before being added again.</param>
/// <param name="stories">StoryCollection containing the stories to add/update
/// in the index</param>
private void AddStoriesToIndex(IndexModifier modifier, bool isIncrementalCrawl, StoryCollection stories)
{
if (isIncrementalCrawl)
{
//remove the stories from the index that have been updated
Log.DebugFormat("Updating index, removing {0} stories", stories.Count);
foreach (Story s in stories)
{
Term existingItem = new Term("id", s.StoryID.ToString());
int j = modifier.DeleteDocuments(existingItem);
}
}
//add the new documents
Log.DebugFormat("Adding batch of {0} stories to the index", stories.Count);
foreach (Story story in stories)
{
//spam stories shouldnt be added to the index
if (story.IsSpam)
continue;
Document doc = new Document();
doc.Add(new Field("url", story.Url, Field.Store.NO, Field.Index.TOKENIZED));
doc.Add(new Field("title", story.Title, Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
doc.Add(new Field("description", story.Description, Field.Store.NO, Field.Index.TOKENIZED));
doc.Add(new Field("users", GetUserWhoKickedSearchString(story), Field.Store.NO, Field.Index.TOKENIZED));
doc.Add(new Field("category", story.Category.Name, Field.Store.NO, Field.Index.TOKENIZED));
doc.Add(new Field("tags", GetStoryTags(story), Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
doc.Add(new Field("id", story.StoryID.ToString(), Field.Store.YES, Field.Index.UN_TOKENIZED));
doc.Add(new Field("kickCount", story.KickCount.ToString(), Field.Store.NO, Field.Index.UN_TOKENIZED));
doc.Add(new Field("dateAdded", DateField.DateToString(story.CreatedOn), Field.Store.NO, Field.Index.UN_TOKENIZED));
modifier.AddDocument(doc);
Log.DebugFormat("StoryId {0} added to index", story.StoryID);
}
}
public void CreateIndexIfNeeded(List<MsDocEntryPoint> documentation, DateTime? lastModified)
{
lock (SynchLock)
{
if (IsIndexingNeeded(lastModified) || !IsOperational)
{
try
{
//Drop all index
if (searcher != null)
searcher.Close();
if (_directory != null)
_directory.Close();
//Delete dir
Directory.Delete(_indexDirectory, true);
_directory = FSDirectory.GetDirectory(_indexDirectory, true); //Reopen directory
var indexModifier = new IndexModifier(_directory, new StandardAnalyzer(), true);
foreach (var entryPoint in documentation)
{
var pointDoc = new Document();
//Id keys
pointDoc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED,
Field.TermVector.YES));
if (!string.IsNullOrEmpty(entryPoint.Summary))
pointDoc.Add(new Field("summary", entryPoint.Summary, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(entryPoint.Example))
pointDoc.Add(new Field("example", entryPoint.Example, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
foreach (var pointMethod in entryPoint.Methods)
{
var doc = new Document();
//Id keys
doc.Add(new Field("point", entryPoint.Name, Field.Store.YES, Field.Index.NOT_ANALYZED,
Field.TermVector.YES));
doc.Add(new Field("path", pointMethod.ToString(), Field.Store.YES,
Field.Index.NOT_ANALYZED, Field.TermVector.YES));
doc.Add(new Field("url", pointMethod.Path, Field.Store.YES, Field.Index.NOT_ANALYZED,
Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Notes))
doc.Add(new Field("notes", pointMethod.Notes, Field.Store.YES, Field.Index.ANALYZED,
Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Remarks))
doc.Add(new Field("remarks", pointMethod.Remarks, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Example))
doc.Add(new Field("examlpe", pointMethod.Example, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Returns))
doc.Add(new Field("returns", pointMethod.Returns, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.ShortName))
doc.Add(new Field("short", pointMethod.ShortName, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Summary))
doc.Add(new Field("summary", pointMethod.Summary, Field.Store.YES,
Field.Index.ANALYZED, Field.TermVector.NO));
foreach (var param in pointMethod.Params)
{
if (!string.IsNullOrEmpty(param.Description))
doc.Add(new Field("param-" + param.Name + "-description", param.Description,
Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
if (!string.IsNullOrEmpty(pointMethod.Remarks))
doc.Add(new Field("param-" + param.Name + "-remarks", param.Remarks,
Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
}
indexModifier.AddDocument(doc);
}
}
indexModifier.Optimize();
indexModifier.Close();
searcher = new IndexSearcher(_directory);
IsOperational = true;
}
catch (Exception)
{
IsOperational = false;
}
}
}
}
/// <summary>
/// Clears the index.
/// </summary>
/// <param name="reCreate">if set to <c>true</c> [re create].</param>
private static void ClearIndex(bool reCreate)
{
if (!Directory.Exists(LuceneFolder))
Directory.CreateDirectory(LuceneFolder);
if (reCreate)
{
IndexModifier modifier = new IndexModifier(LuceneFolder, new StandardAnalyzer(), true);
modifier.Close();
}
}
private void TestIndexInternal(int maxWait)
{
bool create = true;
//Directory rd = new RAMDirectory();
// work on disk to make sure potential lock problems are tested:
System.String tempDir = System.IO.Path.GetTempPath();
if (tempDir == null)
throw new System.IO.IOException("java.io.tmpdir undefined, cannot run test");
System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(tempDir, "lucenetestindex"));
Directory rd = FSDirectory.Open(indexDir);
IndexThread.id = 0;
IndexThread.idStack.Clear();
IndexModifier index = new IndexModifier(rd, new StandardAnalyzer(), create);
IndexThread thread1 = new IndexThread(index, maxWait, 1);
thread1.Start();
IndexThread thread2 = new IndexThread(index, maxWait, 2);
thread2.Start();
while (thread1.IsAlive || thread2.IsAlive)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 100));
}
index.Optimize();
int added = thread1.added + thread2.added;
int deleted = thread1.deleted + thread2.deleted;
Assert.AreEqual(added - deleted, index.DocCount());
index.Close();
try
{
index.Close();
Assert.Fail();
}
catch (System.SystemException e)
{
// expected exception
}
RmDir(indexDir);
}
private static void CloseIndexModifier()
{
IndexModifier.Close();
indexModifier = null;
}
/// <summary>
/// </summary>
protected override void DoIndexNode(Node node, bool reindexAfterRemovingOld)
{
lock(this)
{
foreach (string langKey in luceneDirectoryIndexedByLanguageCodeThenView.Keys)
{
foreach (string viewKey in luceneDirectoryIndexedByLanguageCodeThenView[langKey].Keys)
{
Directory d = luceneDirectoryIndexedByLanguageCodeThenView[langKey][viewKey];
IndexModifier modifier = new IndexModifier(d, LuceneLibrarySearchCriteriaAdaptor.ANALYZER, false);
try
{
foreach (int i in GetDocumentIdsForElementId(node.Id))
{
Document doc = GetDocument(i);
if (MakePath(node) == doc.GetField(LuceneNodeIndexer.PATH_FIELD).StringValue() && node.Order.ToString() == doc.GetField(LuceneNodeIndexer.ORDER_FIELD).StringValue())
{
Debug.WriteLine(
string.Format("DoIndexNode: Removing document {0} for element id {1}", i,
node.Id));
modifier.DeleteDocument(i);
}
}
}
finally
{
Debug.WriteLine("DoIndexNode: Closing index reader");
modifier.Close();
}
if (!reindexAfterRemovingOld) return;
indexWriter = new IndexWriter(d, LuceneLibrarySearchCriteriaAdaptor.ANALYZER, false);
LuceneNodeIndexer indexer = new LuceneNodeIndexer(indexWriter, langKey);
indexer.AddToIndex(node, node.PreferredLabel);
}
}
}
}
