public void Dispose()
{
if (m_MapIndex != null)
{
m_MapIndex.Close();
}
else
{
MapStream?.Close();
}
DataStream?.Close();
IndexReader?.Close();
}
private List <SearchResult> search(string keyword, string shipword)
{
string indexPath = "E:/Index";
//Directory directory = FSDirectory.Open(new File(indexPath));
FSDirectory directory = FSDirectory.Open(new DirectoryInfo(indexPath), new NoLockFactory());
/* bool isUpdate = IndexReader.IndexExists(indexPath);
* if (isUpdate)
* {
* if (IndexWriter.IsLocked(indexPath))
* {
* IndexWriter.Unlock(indexPath);
* }
* }*/
// IndexReader reader = IndexReader.Open(directory, true);
IndexReader reader = DirectoryReader.Open(directory);
IndexSearcher search = new IndexSearcher(reader);
TopDocs topdocs = null;
/* QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29,"content", new PanGuAnalyzer());
* Query query = parser.Parse(keyword);
* TopDocs topdocs = search.Search(query, 1000); */
// 认为地理名词与航母至少有一个不为空
string temp = null;
if ((keyword == "") || (shipword == ""))
{
temp = keyword + shipword;
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_29, "content", new PanGuAnalyzer());
Query query = parser.Parse(temp);
topdocs = search.Search(query, 1000);
}
else if ((keyword != "") && (shipword != ""))
{
//多种field查询
string[] queryString = { kw, shipword }; //地理位置 航母号 时间
string[] fields = { "content", "content" };
BooleanClause.Occur[] flags = { BooleanClause.Occur.MUST, BooleanClause.Occur.MUST };
Query multiQuery = MultiFieldQueryParser.Parse(Lucene.Net.Util.Version.LUCENE_29, queryString, fields, flags, new PanGuAnalyzer());
topdocs = search.Search(multiQuery, 1000);
}
// Query muly = QueryParser.Parse(queryString, fields,flags, new PanGuAnalyzer());
ScoreDoc[] scoreDocs = topdocs.scoreDocs;
List <SearchResult> list = new List <SearchResult>();
logger.Debug("查询结果总数---" + topdocs.totalHits + " 最大的评分--" + topdocs.GetMaxScore());
count = topdocs.totalHits;
// Console.WriteLine("查询结果总数---" + topdocs.totalHits + " 最大的评分--" + topdocs.GetMaxScore());
for (int i = 0; i < scoreDocs.Length; i++)
{
int doc = scoreDocs[i].doc;
Document document = search.Doc(doc);
// Console.WriteLine("id--" + scoreDocs[i].doc + "---scors--" + scoreDocs[i].score + "---uri--" + document.Get("link"));
string number = scoreDocs[i].doc.ToString();
string score = scoreDocs[i].score.ToString();
string uri = document.Get("link");
string date = document.Get("publishtime").ToString();
string title = "标题：" + document.Get("title").ToString() + document.Get("publishtime").ToString();
// SearchResult searcher = new SearchResult() { Number = number, Score = score, BodyPreview = Preview(body, kw) };
SearchResult searcher = new SearchResult()
{
Number = number, Score = score, Uri = uri, Title = title, Date = date
};
list.Add(searcher);
}
reader.Close();
return(list);
}
private void CloseInternalReader(bool trace, IndexReader subReader, bool finalClose)
{
ReaderData readerData;
// TODO: can we avoid the lock?
lock (semaphoreIndexReaderLock)
{
readerData = searchIndexReaderSemaphores[subReader];
}
if (readerData == null)
{
log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory());
// TODO: Should we try to close?
return;
}
// Acquire the locks in the same order as everywhere else
object directoryProviderLock = perDirectoryProviderManipulationLocks[readerData.Provider];
bool closeReader = false;
lock (directoryProviderLock)
{
IndexReader reader;
bool isActive = activeSearchIndexReaders.TryGetValue(readerData.Provider, out reader) &&
reader == subReader;
if (trace)
{
log.Info("IndexReader not active: " + subReader);
}
lock (semaphoreIndexReaderLock)
{
readerData = searchIndexReaderSemaphores[subReader];
if (readerData == null)
{
log.Error("Trying to close a Lucene IndexReader not present: " + subReader.Directory());
// TODO: Should we try to close?
return;
}
//final close, the semaphore should be at 0 already
if (!finalClose)
{
readerData.Semaphore--;
if (trace)
{
log.Info("Semaphore decreased to: " + readerData.Semaphore + " for " + subReader);
}
}
if (readerData.Semaphore < 0)
{
log.Error("Semaphore negative: " + subReader.Directory());
}
if (!isActive && readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(subReader);
closeReader = true;
}
else
{
closeReader = false;
}
}
}
if (closeReader)
{
if (trace)
{
log.Info("Closing IndexReader: " + subReader);
}
try
{
subReader.Close();
}
catch (IOException e)
{
log.Warn("Unable to close Lucene IndexReader", e);
}
}
}
private IndexReader ReplaceActiveReader(IndexReader outOfDateReader, object directoryProviderLock,
IDirectoryProvider directoryProvider, IndexReader[] readers)
{
bool trace = log.IsInfoEnabled;
IndexReader oldReader;
bool closeOldReader = false;
bool closeOutOfDateReader = false;
IndexReader reader;
/**
* Since out of lock protection, can have multiple readers created in //
* not worse than NotShared and limit the locking time, hence scalability
*/
try
{
reader = IndexReader.Open(directoryProvider.Directory);
}
catch (IOException e)
{
throw new SearchException("Unable to open Lucene IndexReader", e);
}
lock (directoryProviderLock)
{
// Since not protected by a lock, other ones can have been added
oldReader = activeSearchIndexReaders[directoryProvider] = reader;
lock (semaphoreIndexReaderLock)
{
searchIndexReaderSemaphores[reader] = new ReaderData(1, directoryProvider);
if (trace)
{
log.Info("Semaphore: 1 for " + reader);
}
if (outOfDateReader != null)
{
ReaderData readerData;
searchIndexReaderSemaphores.TryGetValue(outOfDateReader, out readerData);
if (readerData == null)
{
closeOutOfDateReader = false; //already removed by another prevous thread
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(outOfDateReader);
closeOutOfDateReader = true;
}
else
{
closeOutOfDateReader = false;
}
}
if (oldReader != null && oldReader != outOfDateReader)
{
ReaderData readerData = searchIndexReaderSemaphores[oldReader];
if (readerData == null)
{
log.Warn("Semaphore should not be null");
closeOldReader = true; //TODO should be true or false?
}
else if (readerData.Semaphore == 0)
{
searchIndexReaderSemaphores.Remove(oldReader);
closeOldReader = true;
}
else
{
closeOldReader = false;
}
}
}
}
if (closeOutOfDateReader)
{
if (trace)
{
log.Info("Closing out of date IndexReader " + outOfDateReader);
}
try
{
outOfDateReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
if (closeOldReader)
{
if (trace)
{
log.Info("Closing old IndexReader " + oldReader);
}
try
{
oldReader.Close();
}
catch (IOException e)
{
ReaderProviderHelper.Clean(readers);
throw new SearchException("Unable to close Lucene IndexReader", e);
}
}
return(reader);
}
private void ValidateSearcher(bool forceReopen)
{
EnsureIndex();
if (!forceReopen)
{
if (_reader == null)
{
lock (_locker)
{
//double check
if (_reader == null)
{
try
{
//get a reader - could be NRT or based on directly depending on how this was constructed
_reader = _nrtWriter == null
? OpenNewReader()
: _nrtWriter.GetReader();
_searcher = new IndexSearcher(_reader);
//track it!
OpenReaderTracker.Current.AddOpenReader(_reader);
}
catch (IOException ex)
{
throw new ApplicationException("Could not create an index searcher with the supplied lucene directory", ex);
}
}
}
}
else
{
switch (_reader.GetReaderStatus())
{
case ReaderStatus.Current:
break;
case ReaderStatus.Closed:
lock (_locker)
{
//get a reader - could be NRT or based on directly depending on how this was constructed
_reader = _nrtWriter == null
? OpenNewReader()
: _nrtWriter.GetReader();
_searcher = new IndexSearcher(_reader);
//track it!
OpenReaderTracker.Current.AddOpenReader(_reader);
}
break;
case ReaderStatus.NotCurrent:
lock (_locker)
{
//yes, this is actually the way the Lucene wants you to work...
//normally, i would have thought just calling Reopen() on the underlying reader would suffice... but it doesn't.
//here's references:
// http://stackoverflow.com/questions/1323779/lucene-indexreader-reopen-doesnt-seem-to-work-correctly
// http://gist.github.com/173978
//Also note that when a new reader is returned from Reopen() the old reader is not actually closed -
// but more importantly the old reader might still be in use from another thread! So we can't just
// close it here because that would cause a YSOD: Lucene.Net.Store.AlreadyClosedException: this IndexReader is closed
// since another thread might be using it. I'm 'hoping' that the GC will just take care of the left over reader's that might
// be currently being used in a search, otherwise there's really no way to now when it's safe to close the reader.
var newReader = _reader.Reopen();
if (newReader != _reader)
{
//if it's changed, then re-assign, note: the above, before we used to close the old one here
// but that will cause problems since the old reader might be in use on another thread.
_reader = newReader;
_searcher = new IndexSearcher(_reader);
//track it!
OpenReaderTracker.Current.AddOpenReader(_reader);
//get rid of old ones (anything a minute or older)
OpenReaderTracker.Current.CloseStaleReaders(GetLuceneDirectory(), TimeSpan.FromMinutes(1));
}
}
break;
}
}
}
else
{
if (_reader != null)
{
lock (_locker)
{
//double check
if (_reader != null)
{
try
{
_searcher.Close();
_reader.Close();
}
catch (IOException ex)
{
//this will happen if it's already closed ( i think )
Trace.TraceError("Examine: error occurred closing index searcher. {0}", ex);
}
finally
{
//set to null in case another call to this method has passed the first lock and is checking for null
_searcher = null;
_reader = null;
}
try
{
//get a reader - could be NRT or based on directly depending on how this was constructed
_reader = _nrtWriter == null
? OpenNewReader()
: _nrtWriter.GetReader();
_searcher = new IndexSearcher(_reader);
}
catch (IOException ex)
{
throw new ApplicationException("Could not create an index searcher with the supplied lucene directory", ex);
}
}
}
}
}
}
public void InternalSharding()
{
ISession s = OpenSession();
ITransaction tx = s.BeginTransaction();
Animal a = new Animal();
a.Id = 1;
a.Name = "Elephant";
s.Persist(a);
a = new Animal();
a.Id = 2;
a.Name = "Bear";
s.Persist(a);
tx.Commit();
s.Clear();
IndexReader reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal00"));
try
{
int num = reader.NumDocs();
Assert.AreEqual(1, num);
}
finally
{
reader.Close();
}
reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal.1"));
try
{
int num = reader.NumDocs();
Assert.AreEqual(1, num);
}
finally
{
reader.Close();
}
tx = s.BeginTransaction();
a = (Animal)s.Get(typeof(Animal), 1);
a.Name = "Mouse";
tx.Commit();
s.Clear();
reader = IndexReader.Open(new FileInfo(BaseIndexDir.FullName + "\\Animal.1"));
try
{
int num = reader.NumDocs();
Assert.AreEqual(1, num);
TermDocs docs = reader.TermDocs(new Term("name", "mouse"));
Assert.IsTrue(docs.Next());
Document doc = reader.Document(docs.Doc());
Assert.IsFalse(docs.Next());
}
finally
{
reader.Close();
}
tx = s.BeginTransaction();
IFullTextSession fts = Search.CreateFullTextSession(s);
QueryParser parser = new QueryParser("id", new StopAnalyzer());
IList results = fts.CreateFullTextQuery(parser.Parse("name:mouse OR name:bear")).List();
Assert.AreEqual(2, results.Count, "Either double insert, single update, or query fails with shards");
// cleanup
s.Delete("from System.Object");
tx.Commit();
s.Close();
}
public static TermInfo[] GetHighFreqTerms(Directory dir,
Hashtable junkWords,
int numTerms,
String[] fields)
{
if (dir == null || fields == null)
{
return(new TermInfo[0]);
}
IndexReader reader = IndexReader.Open(dir, true);
TermInfoQueue tiq = new TermInfoQueue(numTerms);
TermEnum terms = reader.Terms();
int minFreq = 0;
while (terms.Next())
{
String field = terms.Term().Field();
if (fields != null && fields.Length > 0)
{
bool skip = true;
for (int i = 0; i < fields.Length; i++)
{
if (field.Equals(fields[i]))
{
skip = false;
break;
}
}
if (skip)
{
continue;
}
}
if (junkWords != null && junkWords[terms.Term().Text()] != null)
{
continue;
}
if (terms.DocFreq() > minFreq)
{
TermInfo top = (TermInfo)tiq.Add(new TermInfo(terms.Term(), terms.DocFreq()));
if (tiq.Size() >= numTerms) // if tiq overfull
{
tiq.Pop(); // remove lowest in tiq
minFreq = top.DocFreq; // reset minFreq
}
}
}
TermInfo[] res = new TermInfo[tiq.Size()];
for (int i = 0; i < res.Length; i++)
{
res[res.Length - i - 1] = (TermInfo)tiq.Pop();
}
reader.Close();
return(res);
}
// tests storing "id" and "field2" fields as pulsing codec,
// whose term sort is backwards unicode code point, and
// storing "field1" as a custom entirely-in-RAM codec
public virtual void TestPerFieldCodec()
{
int NUM_DOCS = atLeast(173);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
BaseDirectoryWrapper dir = newDirectory();
dir.CheckIndexOnClose = false; // we use a custom codec provider
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(new CustomPerFieldCodec()).setMergePolicy(newLogMergePolicy(3)));
Document doc = new Document();
// uses default codec:
doc.Add(newTextField("field1", "this field uses the standard codec as the test", Field.Store.NO));
// uses pulsing codec:
Field field2 = newTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO);
doc.Add(field2);
Field idField = newStringField("id", "", Field.Store.NO);
doc.Add(idField);
for (int i = 0; i < NUM_DOCS; i++)
{
idField.StringValue = "" + i;
w.addDocument(doc);
if ((i + 1) % 10 == 0)
{
w.commit();
}
}
if (VERBOSE)
{
Console.WriteLine("TEST: now delete id=77");
}
w.deleteDocuments(new Term("id", "77"));
IndexReader r = DirectoryReader.Open(w, true);
Assert.AreEqual(NUM_DOCS - 1, r.NumDocs());
IndexSearcher s = newSearcher(r);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
r.Close();
if (VERBOSE)
{
Console.WriteLine("\nTEST: now delete 2nd doc");
}
w.deleteDocuments(new Term("id", "44"));
if (VERBOSE)
{
Console.WriteLine("\nTEST: now force merge");
}
w.forceMerge(1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now open reader");
}
r = DirectoryReader.Open(w, true);
Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc());
Assert.AreEqual(NUM_DOCS - 2, r.NumDocs());
s = newSearcher(r);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now close NRT reader");
}
r.Close();
w.Dispose();
dir.Dispose();
}
public override void TearDown()
{
reader.Close();
base.TearDown();
}
void IDisposable.Dispose()
{
GC.SuppressFinalize(this);
_idx.Close();
}
private void ProcessQueue(DataTable q, string indexPath)
{
rowsProcessed = 0;
rowsToProcess = q.Rows.Count;
// first process deletes with reader
try
{
IndexReader reader = IndexReader.Open(indexPath);
foreach (DataRow row in q.Rows)
{
Term term = new Term("Key", row["ItemKey"].ToString());
try
{
reader.DeleteDocuments(term);
log.Debug("reader.DeleteDocuments(term) for Key " + row["ItemKey"].ToString());
}
catch (Exception ge)
{
// TODO: monitor what real exceptions if any occur and then
// change this catch to catch only the expected ones
// instead of non specific exception
log.Error(ge);
}
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (removeOnly)
{
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
reader.Close();
}
catch (IOException ex)
{
log.Info("IndexWriter swallowed exception this is not unexpected if building or rebuilding the search index ", ex);
errorCount += 1;
}
catch (TypeInitializationException ex)
{
log.Info("IndexWriter swallowed exception ", ex);
errorCount += 1;
}
// next add items with writer
IndexWriter indexWriter = GetWriter(indexPath);
if (indexWriter == null)
{
log.Error("failed to get IndexWriter for path: " + indexPath);
errorCount += 1;
return;
}
foreach (DataRow row in q.Rows)
{
bool removeOnly = Convert.ToBoolean(row["RemoveOnly"]);
if (!removeOnly)
{
try
{
IndexItem indexItem
= (IndexItem)SerializationHelper.DeserializeFromString(typeof(IndexItem), row["SerializedItem"].ToString());
Document doc = GetDocument(indexItem);
WriteToIndex(doc, indexWriter);
log.Debug("called WriteToIndex(doc, indexWriter) for key " + indexItem.Key);
Int64 rowId = Convert.ToInt64(row["RowId"]);
IndexingQueue.Delete(rowId);
}
catch (Exception ex)
{
log.Error(ex);
}
}
if (DateTime.UtcNow > nextStatusUpdateTime)
{
// don't mark as complete because there may be more qu items
//for different index paths in a multi site installation
bool markAsComplete = false;
ReportStatus(markAsComplete);
}
}
try
{
indexWriter.Optimize();
}
catch (IOException ex)
{
log.Error(ex);
}
try
{
indexWriter.Close();
}
catch (IOException ex)
{
log.Error(ex);
}
}
public void _Search()
{
//string request = (searchParams as string);
string old_request = "";
string new_request = "";
while (true)
{
lock (_request)
{
new_request = _request;
}
if (new_request != old_request)
{
old_request = new_request;
if (new_request.Length != 0)
{
IndexReader reader = null;
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
string index_path = Owl.Properties.Settings.Default.IndexPath;
try
{
reader = IndexReader.Open(index_path);
}
catch
{
status.SearchStatus = string.Format("Problems while opening Index: has it been created in {0} ?", Owl.Properties.Settings.Default.IndexPath);
}
int nb_docs = 0;
int found_docs = 0;
if (reader != null)
{
try
{
Searcher searcher = new IndexSearcher(reader);
Analyzer analyzer = new StandardAnalyzer();
//QueryParser parser = new QueryParser("contents", analyzer);
MultiFieldQueryParser parser = new MultiFieldQueryParser(new string[] { "contents", "path" }, analyzer);
Query query = parser.Parse(new_request);
SimpleCall sc = delegate() { resultItems.Clear(); };
Application.Current.Dispatcher.Invoke(DispatcherPriority.Background, sc);
//Hits hits = searcher.Search(query);
TopDocs docs = searcher.Search(query, null, 100);
int num_doc = 1;
foreach (ScoreDoc score_doc in docs.scoreDocs)
{
if (HasRequestChanged(new_request))
{
break;
}
Document doc = searcher.Doc(score_doc.doc);
System.String path = doc.Get("path");
//
SimpleCall sc2 = delegate()
{
resultItems.Add(new Result(string.Format("{0} - {2} ({1})%\n{3}",
num_doc++, (int)((score_doc.score * 100) / docs.GetMaxScore()),
System.IO.Path.GetFileName(path), System.IO.Path.GetDirectoryName(path)), path));
};
Application.Current.Dispatcher.Invoke(DispatcherPriority.Background, sc2);
//
}
found_docs = docs.scoreDocs.Length;
nb_docs = reader.NumDocs();
searcher.Close();
}
//catch (TokenMgrError)
//{ }
catch (Exception e)
{
status.SearchStatus = string.Format("Problems with request {0} ", new_request);
Log.Error(e);
}
finally
{
reader.Close();
stopWatch.Stop();
}
}
//---
status.SearchStatus = string.Format("{0} results for '{3}' in {1} docs (took {2} ms)", found_docs,
nb_docs, stopWatch.ElapsedMilliseconds, new_request);
}
}
else
{
_mre.Reset();
_mre.WaitOne();
}
//Thread.Sleep(250);
}
}
public void Close()
{
_reader.Close();
}
internal void StartCrawler()
{
// install the scheduler in a separate thread
new Thread(delegate()
{
#if Log
Console.WriteLine("Initiating Resource Scheduler in file system crawler");
#endif
Scheduler();
}).Start();
//Start the crawler in the current thread
#if Log
Console.WriteLine("Initiating file system Crawler");
#endif
long start = DateTime.Now.Ticks;
foreach (string drive in GlobalData.Drives)
{
DriveInfo di = new DriveInfo(drive);
if (di.IsReady && di.DriveType == DriveType.Fixed && GlobalData.RunCrawler)
{
//check whether the index directory is clean or not. if not delete all files in it
if (Directory.Exists(GlobalData.IndexRootPath + drive[0]))
{
Directory.Delete(GlobalData.IndexRootPath + drive[0], true);
}
Directory.CreateDirectory(GlobalData.IndexRootPath + drive[0]);
//Initialize the indexer
GlobalData.Indexer = new EDSIndexer(GlobalData.IndexRootPath + drive[0]);
//Start File System Crawler
Crawler(drive);
//Optimize the indexes
GlobalData.Indexer.Optimize();
GlobalData.Indexer.Close();
}
}
#if Log
Console.WriteLine("Completed File System Crawling");
#endif
if (GlobalData.RunCrawler) //only when the crawler finished its work do the following stuff.....otherwise if crawler has been terminated by closing the application donot do the following stuff
{
GlobalData.IndexingCompleted = true;
long end = DateTime.Now.Ticks;
DateTime dt = new DateTime(end - start);
GlobalData.lIndexingStatus.Text = "Indexing: Completed 100% in " + (dt.Hour != 0 ? dt.Hour + "h:" : "") + (dt.Minute != 0 ? dt.Minute + "m:" : "") + (dt.Second != 0 ? dt.Second + "s: " : "") + (dt.Millisecond != 0 ? dt.Millisecond + "ms" : "");
//update the statusbar indexed documents count
int docCount = 0;
foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath))
{
IndexReader ir = IndexReader.Open(dir);
docCount += ir.NumDocs();
ir.Close();
}
if (GlobalData.MainStatusStrip != null && GlobalData.MainStatusStrip.Items.Count != 0)
{
GlobalData.MainStatusStrip.Items["tsslDocCount"].Text = docCount + " documents indexed ";
}
//Finally start file system monitor for further file system updates
if (GlobalData.RunMonitor)
{
#if Log
Console.WriteLine("Starting File System Monitor");
#endif
FileSystemMonitor fsw = FileSystemMonitor.GetInstance();
foreach (string drive in Environment.GetLogicalDrives())
{
fsw.AddMonitor(drive);
}
fsw.StartAllMonitors();
}
}
//after completion of crawling stop the scheduler by setting RunScheduler => false;
GlobalData.RunScheduler = false;
}
public List <Post> Similar(int postid, int itemsToReturn)
{
List <Post> TList = new List <Post>();
int docId = -1;
IndexSearcher searcher = null;
IndexReader reader = null;
if (rd == null)
{
BuildIndex();
}
lck.AcquireReaderLock(ReaderTimeOut);
try
{
Analyzer analyzer = GetAnalyzer();
QueryParser parser = GetQueryParser(analyzer);
parser.SetDefaultOperator(QueryParser.AND_OPERATOR);
Query q = parser.Parse("postid:" + postid);
searcher = new IndexSearcher(rd, true);
//TODO
#pragma warning disable CS0618 // Type or member is obsolete
Hits hits = searcher.Search(q);
#pragma warning restore CS0618 // Type or member is obsolete
if (hits != null && hits.Length() > 0)
{
docId = hits.Id(0);
}
if (docId > -1)
{
reader = IndexReader.Open(rd, true);
TermFreqVector tfv = reader.GetTermFreqVector(docId, "exact");
BooleanQuery booleanQuery = new BooleanQuery();
for (int j = 0; j < tfv.Size(); j++)
{
TermQuery tq = new TermQuery(new Term("exact", tfv.GetTerms()[j]));
booleanQuery.Add(tq, BooleanClause.Occur.SHOULD);
}
//TODO
#pragma warning disable CS0618 // Type or member is obsolete
Hits similarhits = searcher.Search(booleanQuery, Sort.RELEVANCE);
#pragma warning restore CS0618 // Type or member is obsolete
for (int i = 0; i < similarhits.Length(); i++)
{
Document doc = similarhits.Doc(i);
if (similarhits.Id(i) != docId)
{
TList.Add(CreateFromDocument(doc, analyzer, null));
}
if (TList.Count >= itemsToReturn)
{
break;
}
}
}
}
catch (Exception)
{
}
finally
{
if (searcher != null)
{
searcher.Close();
}
if (reader != null)
{
reader.Close();
}
lck.ReleaseReaderLock();
}
return(TList);
}
public void End(bool shouldClose)
{
if (!_is_started)
{
return;
}
if (!shouldClose)
{
return;
}
//build 2del file list
if (!_job_status.Cancelled)
{
TermEnum term_enum = _index_reader.Terms();
Term path_term = new Term("path");
int nb_terms = 0;
while (term_enum.SkipTo(path_term)) //skip to new term equal or *ABOVE* "path:" !!!
{
Term term = term_enum.Term();
if (term.Field() != path_term.Field())
{
break;
}
if (!File.Exists(term.Text()))
{
_del_file_list.Add(term.Text());
}
if (_job_status.Cancelled)
{
break;
}
nb_terms++;
}
term_enum.Close();
Logger.Log.Info("update: deletion: {0} analyzed terms, found {1} vanished files.", nb_terms, _del_file_list.Count);
}
_index_searcher.Close();
_index_reader.Close();
//--- deleting deprecated
if ((_del_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
int num_file = 0;
int nb_files = _del_file_list.Count;
IndexWriter writer = new IndexWriter(_index_path, _default_analyzer, false);
foreach (string path in _del_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: removing (from index) file {0}/{1} - {2}", num_file, _del_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.DeleteDocuments(new Term("path", path));
}
writer.Commit();
writer.Close();
watch.Stop();
}
//adding new files
if ((_add_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
IndexWriter writer = null;
try
{
writer = new IndexWriter(_index_path, _default_analyzer, false, new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
int num_file = 0;
int nb_files = _add_file_list.Count;
foreach (BasicFileInfo fi in _add_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: indexing new file {0}/{1} - {2}", num_file, _add_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.AddDocument(_doc_factory.CreateFromPath(fi.FilePath, fi.LastModification));
if (num_file % 20 == 0)
{
writer.Commit();
}
}
writer.Commit();
}
catch (System.Exception ex)
{
Log.Error(ex);
}
finally
{
if (writer != null)
{
writer.Close();
writer = null;
}
}
watch.Stop();
}
//updating modified files
if ((_upd_file_list.Count > 0) && (!_job_status.Cancelled))
{
Stopwatch watch = new Stopwatch();
watch.Start();
int num_file = 0;
int nb_files = _upd_file_list.Count;
IndexWriter writer = null;
try
{
writer = new IndexWriter(_index_path, _default_analyzer, false,
new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
foreach (BasicFileInfo fi in _upd_file_list)
{
if (((num_file++) % 101) == 1)
{
int progress = ((((num_file++) + 1)) * 100) / nb_files;
_job_status.Progress = progress;
_job_status.Description = String.Format("upd: modified file {0}/{1} - {2}", num_file, _upd_file_list.Count,
StringFu.TimeSpanToString(new TimeSpan((long)(watch.ElapsedMilliseconds) * 10000)));
}
if (_job_status.Cancelled)
{
break;
}
writer.UpdateDocument(new Term("path", fi.FilePath),
_doc_factory.CreateFromPath(fi.FilePath, fi.LastModification));
}
writer.Commit();
//LittleBeagle.Properties.Settings.Default.NbIndexedFiles = num_file;
}
catch (System.Exception ex)
{
Log.Error(ex);
}
finally
{
if (writer != null)
{
writer.Close();
writer = null;
}
}
watch.Stop();
}
}
void IndexGatheredFiles(object state)
{
try
{
#region Deleting files from the indexes
//Iterate over all the directories present in the %appdata%\EDS\indexes folder
if (GlobalData.DeletedFiles.Count > 0)
{
foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath))
{
//Deletion of files from indexes
IndexReader reader = IndexReader.Open(dir);
lock (GlobalData.DeletedFiles)
{
//foreach (string path in GlobalData.DeletedFiles)
while (GlobalData.DeletedFiles.Count != 0)
{
reader.DeleteDocuments(new Term("path", GlobalData.DeletedFiles[0]));
GlobalData.lIndexingStatus.Text = GlobalData.DeletedFiles[0] + " is deleted from indexes";
GlobalData.DeletedFiles.RemoveAt(0);
}
}
reader.Close();
}
}
#endregion
#region indexing newly created files
//Indexing of new Files
if (GlobalData.CreatedFiles.Count > 0)
{
lock (GlobalData.CreatedFiles)
{
// foreach (string path in GlobalData.CreatedFiles)
while (GlobalData.CreatedFiles.Count != 0)
{
string path = GlobalData.CreatedFiles[0];
//Get the respective content handler. If no content handler is present return the default handler
IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ?
GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"];
if (parser != null) //For some nasty reason the file got deleted or doesnot exist
{
StringDictionary properties = parser.GetProperties(path);
if (properties != null)
{
//Initialize the indexer
EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]);
indexer.Index(properties);
indexer.Close();
}
}
GlobalData.lIndexingStatus.Text = GlobalData.CreatedFiles[0] + " is newly indexed";
GlobalData.CreatedFiles.RemoveAt(0);
}
}
}
#endregion
#region Updating already indexed files
lock (GlobalData.ChangedFiles)
{
//foreach (string path in GlobalData.ChangedFiles)
while (GlobalData.ChangedFiles.Count != 0)
{
string path = GlobalData.ChangedFiles[0];
//first delete that document
foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath))
{
//Deletion of files from indexes
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocuments(new Term("path", path));
bool deleted = reader.HasDeletions();
reader.Close();
if (deleted)
{
break;
}
}
//reindex the document
//Get the respective content handler. If no content handler is present return the default handler
IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ?
GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"];
if (parser != null) //For some nasty reason the file got deleted or doesnot exist
{
StringDictionary properties = parser.GetProperties(path);
if (properties != null)
{
//Initialize the indexer
EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]);
indexer.Index(properties);
indexer.Close();
}
}
GlobalData.lIndexingStatus.Text = path + " is updated in the indexes";
GlobalData.ChangedFiles.RemoveAt(0);
}
}
#endregion
#region Updating renamed files
lock (GlobalData.RenamedFiles)
{
//foreach (string path in GlobalData.ChangedFiles)
while (GlobalData.RenamedFiles.Count != 0)
{
string path = GlobalData.RenamedFiles[0].OldFullPath;
//first delete the old document
foreach (string dir in Directory.GetDirectories(GlobalData.IndexRootPath))
{
//Deletion of files from indexes
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocuments(new Term("path", path));
bool deleted = reader.HasDeletions();
reader.Close();
if (deleted)
{
break;
}
}
//reindex the new document
//Get the respective content handler. If no content handler is present return the default handler
path = GlobalData.RenamedFiles[0].FullPath;
IEDSParser parser = GlobalData.Parsers.ContainsKey(Path.GetExtension(path).ToLower()) ?
GlobalData.Parsers[Path.GetExtension(path).ToLower()] : GlobalData.Parsers["*.*"];
if (parser != null) //For some nasty reason the file got deleted or doesnot exist
{
StringDictionary properties = parser.GetProperties(path);
if (properties != null)
{
//Initialize the indexer
EDSIndexer indexer = new EDSIndexer(GlobalData.IndexRootPath + path[0]);
indexer.Index(properties);
indexer.Close();
}
}
GlobalData.lIndexingStatus.Text = "Renamed file " + path + " is updated in the indexes";
GlobalData.RenamedFiles.RemoveAt(0);
}
}
#endregion
}
catch (Exception ex)
{
#if Log
Console.WriteLine(ex);
#endif
}
}
public void TearDown()
{
reader.Close();
searcher.Close();
directory.Close();
}
protected internal virtual IndexReader DoReopen(bool doClone)
{
EnsureOpen();
bool reopened = false;
IList <IndexReader> newReaders = new List <IndexReader>();
bool success = false;
try
{
foreach (var oldReader in readers)
{
IndexReader newReader = null;
if (doClone)
{
newReader = (IndexReader)oldReader.Clone();
}
else
{
newReader = oldReader.Reopen();
}
newReaders.Add(newReader);
// if at least one of the subreaders was updated we remember that
// and return a new ParallelReader
if (newReader != oldReader)
{
reopened = true;
}
}
success = true;
}
finally
{
if (!success && reopened)
{
for (int i = 0; i < newReaders.Count; i++)
{
IndexReader r = newReaders[i];
if (r != readers[i])
{
try
{
r.Close();
}
catch (System.IO.IOException)
{
// keep going - we want to clean up as much as possible
}
}
}
}
}
if (reopened)
{
List <bool> newDecrefOnClose = new List <bool>();
ParallelReader pr = new ParallelReader();
for (int i = 0; i < readers.Count; i++)
{
IndexReader oldReader = readers[i];
IndexReader newReader = newReaders[i];
if (newReader == oldReader)
{
newDecrefOnClose.Add(true);
newReader.IncRef();
}
else
{
// this is a new subreader instance, so on close() we don't
// decRef but close it
newDecrefOnClose.Add(false);
}
pr.Add(newReader, !storedFieldReaders.Contains(oldReader));
}
pr.decrefOnClose = newDecrefOnClose;
pr.incRefReaders = incRefReaders;
return(pr);
}
else
{
// No subreader was refreshed
return(this);
}
}
public override void TearDown()
{
reader.Close();
dir2.Close();
}
private void CloseIndexReader(IndexReader myReader)
{
myReader.Close();
myReader.Dispose();
}
