| public SetMergeScheduler ( MergeScheduler mergeScheduler ) : void | ||
| mergeScheduler | MergeScheduler | |
| return | void | |
public virtual void TestNoExtraFiles()
{
RAMDirectory directory = new MockRAMDirectory();
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = pass == 0;
IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
for (int iter = 0; iter < 7; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
for (int j = 0; j < 21; j++)
{
Document doc = new Document();
doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles autoCommit=" + autoCommit);
// Reopen
writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
}
writer.Close();
}
directory.Close();
}
public virtual void TestNoExtraFiles()
{
RAMDirectory directory = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
for (int iter = 0; iter < 7; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
for (int j = 0; j < 21; j++)
{
Document doc = new Document();
doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles");
// Reopen
writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED);
}
writer.Close();
directory.Close();
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
int extraCount = 0;
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i * 20 + j));
writer.AddDocument(doc);
}
while (true)
{
// must cycle here because sometimes the merge flushes
// the doc we just added and so there's nothing to
// flush, and we don't hit the exception
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush(true, false, true);
if (failure.hitExc)
{
Assert.Fail("failed to hit IOException");
}
extraCount++;
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
break;
}
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory, true);
Assert.AreEqual(200 + extraCount, reader.NumDocs());
reader.Close();
directory.Close();
}
public virtual void TestNoWaitClose()
{
RAMDirectory directory = new MockRAMDirectory();
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = pass == 0;
IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
for (int iter = 0; iter < 10; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(100);
for (int j = 0; j < 201; j++)
{
idField.SetValue(System.Convert.ToString(iter * 201 + j));
writer.AddDocument(doc);
}
int delID = iter * 201;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(new Term("id", System.Convert.ToString(delID)));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
writer.SetMergeFactor(3);
writer.AddDocument(doc);
writer.Flush();
writer.Close(false);
IndexReader reader = IndexReader.Open(directory);
Assert.AreEqual((1 + iter) * 182, reader.NumDocs());
reader.Close();
// Reopen
writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
}
writer.Close();
}
directory.Close();
}
/*
* Run one indexer and 2 searchers against single index as
* stress test.
*/
public virtual void RunStressTest(Directory directory, MergeScheduler mergeScheduler)
{
IndexWriter modifier = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED, null);
modifier.SetMaxBufferedDocs(10);
TimedThread[] threads = new TimedThread[4];
int numThread = 0;
if (mergeScheduler != null)
{
modifier.SetMergeScheduler(mergeScheduler, null);
}
// One modifier that writes 10 docs then removes 5, over
// and over:
IndexerThread indexerThread = new IndexerThread(this, modifier, threads);
threads[numThread++] = indexerThread;
indexerThread.Start();
IndexerThread indexerThread2 = new IndexerThread(this, modifier, threads);
threads[numThread++] = indexerThread2;
indexerThread2.Start();
// Two searchers that constantly just re-instantiate the
// searcher:
SearcherThread searcherThread1 = new SearcherThread(directory, threads);
threads[numThread++] = searcherThread1;
searcherThread1.Start();
SearcherThread searcherThread2 = new SearcherThread(directory, threads);
threads[numThread++] = searcherThread2;
searcherThread2.Start();
for (int i = 0; i < numThread; i++)
{
threads[i].Join();
}
modifier.Close();
for (int i = 0; i < numThread; i++)
{
Assert.IsTrue(!((TimedThread)threads[i]).failed);
}
//System.out.println(" Writer: " + indexerThread.count + " iterations");
//System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
//System.out.println("Searcher 2: " + searcherThread2.count + " searchers created");
}
public void Initialize()
{
directory = FSDirectory.Open(new DirectoryInfo(path));
if (IndexWriter.IsLocked(directory))
IndexWriter.Unlock(directory);
analyzer = new LowerCaseKeywordAnalyzer();
writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
int extraCount = 0;
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i*20 + j));
writer.AddDocument(doc);
}
while (true)
{
// must cycle here because sometimes the merge flushes
// the doc we just added and so there's nothing to
// flush, and we don't hit the exception
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush(true, false, true);
if (failure.hitExc)
Assert.Fail("failed to hit IOException");
extraCount++;
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
break;
}
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory, true);
Assert.AreEqual(200 + extraCount, reader.NumDocs());
reader.Close();
directory.Close();
}
public void Initialize()
{
if (System.IO.Directory.Exists(path) == false)
System.IO.Directory.CreateDirectory(path);
directory = FSDirectory.Open(new DirectoryInfo(path));
if (IndexWriter.IsLocked(directory))
IndexWriter.Unlock(directory);
analyzer = new LowerCaseKeywordAnalyzer();
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
}
public virtual void TestNoWaitClose()
{
RAMDirectory directory = new MockRAMDirectory();
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED, null);
for (int iter = 0; iter < 10; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms, null);
writer.SetMaxBufferedDocs(2);
writer.MergeFactor = 100;
for (int j = 0; j < 201; j++)
{
idField.SetValue(System.Convert.ToString(iter * 201 + j));
writer.AddDocument(doc, null);
}
int delID = iter * 201;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(null, new Term("id", delID.ToString()));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
writer.MergeFactor = 3;
writer.AddDocument(doc, null);
writer.Commit(null);
writer.Close(false);
IndexReader reader = IndexReader.Open((Directory)directory, true, null);
Assert.AreEqual((1 + iter) * 182, reader.NumDocs());
reader.Close();
// Reopen
writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED, null);
}
writer.Close();
directory.Close();
}
public virtual void TestDeleteMerging()
{
RAMDirectory directory = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
LogDocMergePolicy mp = new LogDocMergePolicy(writer);
writer.SetMergePolicy(mp);
// Force degenerate merging so we can get a mix of
// merging of segments with and without deletes at the
// start:
mp.MinMergeDocs = 1000;
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 100; j++)
{
idField.SetValue(System.Convert.ToString(i * 100 + j));
writer.AddDocument(doc);
}
int delID = i;
while (delID < 100 * (1 + i))
{
writer.DeleteDocuments(new Term("id", "" + delID));
delID += 10;
}
writer.Commit();
}
writer.Close();
IndexReader reader = IndexReader.Open(directory, true);
// Verify that we did not lose any deletes...
Assert.AreEqual(450, reader.NumDocs());
reader.Close();
directory.Close();
}
public virtual void TestHangOnClose()
{
Directory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergePolicy(new LogByteSizeMergePolicy(writer));
writer.SetMaxBufferedDocs(5);
writer.SetUseCompoundFile(false);
writer.SetMergeFactor(100);
Document doc = new Document();
doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 60; i++)
{
writer.AddDocument(doc);
}
writer.SetMaxBufferedDocs(200);
Document doc2 = new Document();
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
for (int i = 0; i < 10; i++)
{
writer.AddDocument(doc2);
}
writer.Close();
Directory dir2 = new MockRAMDirectory();
writer = new IndexWriter(dir2, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(writer);
lmp.SetMinMergeMB(0.0001);
writer.SetMergePolicy(lmp);
writer.SetMergeFactor(4);
writer.SetUseCompoundFile(false);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.AddIndexesNoOptimize(new Directory[] { dir });
writer.Close();
dir.Close();
dir2.Close();
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i * 20 + j));
writer.AddDocument(doc);
}
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush();
Assert.Fail("failed to hit IOException");
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory);
Assert.AreEqual(200, reader.NumDocs());
reader.Close();
directory.Close();
}
public virtual void TestFlushExceptions()
{
MockRAMDirectory directory = new MockRAMDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush();
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 20; j++)
{
idField.SetValue(System.Convert.ToString(i * 20 + j));
writer.AddDocument(doc);
}
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush();
Assert.Fail("failed to hit IOException");
}
catch (System.IO.IOException ioe)
{
failure.ClearDoFail();
}
}
writer.Close();
IndexReader reader = IndexReader.Open(directory);
Assert.AreEqual(200, reader.NumDocs());
reader.Close();
directory.Close();
}
/// <summary> Close the IndexReader and open an IndexWriter.</summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<code>write.lock</code> could not
/// be obtained)
/// </summary>
/// <throws> IOException if there is a low-level IO error </throws>
protected internal virtual void CreateIndexWriter()
{
if (indexWriter == null)
{
if (indexReader != null)
{
indexReader.Close();
indexReader = null;
}
indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength));
// IndexModifier cannot use ConcurrentMergeScheduler
// because it synchronizes on the directory which can
// cause deadlock
indexWriter.SetMergeScheduler(new SerialMergeScheduler());
indexWriter.SetInfoStream(infoStream);
indexWriter.SetUseCompoundFile(useCompoundFile);
if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH)
{
indexWriter.SetMaxBufferedDocs(maxBufferedDocs);
}
indexWriter.SetMergeFactor(mergeFactor);
}
}
private static IndexWriter CreateIndexWriter(Directory directory)
{
var indexWriter = new IndexWriter(directory, new StopAnalyzer(Version.LUCENE_29), IndexWriter.MaxFieldLength.UNLIMITED);
using (indexWriter.MergeScheduler){}
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
// RavenDB already manages the memory for those, no need for Lucene to do this as well
indexWriter.MergeFactor = 1024;
indexWriter.SetMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
indexWriter.SetRAMBufferSizeMB(1024);
return indexWriter;
}
public virtual void TestTermVectorCorruption3()
{
Directory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, false, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy());
Document document = new Document();
document = new Document();
Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
document.Add(storedField);
Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
document.Add(termVectorField);
for (int i = 0; i < 10; i++)
writer.AddDocument(document);
writer.Close();
writer = new IndexWriter(dir, false, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy());
for (int i = 0; i < 6; i++)
writer.AddDocument(document);
writer.Optimize();
writer.Close();
IndexReader reader = IndexReader.Open(dir);
for (int i = 0; i < 10; i++)
{
reader.GetTermFreqVectors(i);
reader.Document(i);
}
reader.Close();
dir.Close();
}
public virtual void TestMergeCompressedFields()
{
System.IO.FileInfo indexDir = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "mergecompressedfields"));
Directory dir = FSDirectory.Open(indexDir);
try
{
for (int i = 0; i < 5; i++)
{
// Must make a new writer & doc each time, w/
// different fields, so bulk merge of stored fields
// cannot run:
IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), i == 0, IndexWriter.MaxFieldLength.UNLIMITED);
try
{
w.SetMergeFactor(5);
w.SetMergeScheduler(new SerialMergeScheduler());
Document doc = new Document();
doc.Add(new Field("test1", "this is some data that will be compressed this this this", Field.Store.COMPRESS, Field.Index.NO));
doc.Add(new Field("test2", new byte[20], Field.Store.COMPRESS));
doc.Add(new Field("field" + i, "random field", Field.Store.NO, Field.Index.ANALYZED));
w.AddDocument(doc);
}
finally
{
w.Close();
}
}
byte[] cmp = new byte[20];
IndexReader r = IndexReader.Open(dir);
try
{
for (int i = 0; i < 5; i++)
{
Document doc = r.Document(i);
Assert.AreEqual(doc.GetField("test1").StringValue(), "this is some data that will be compressed this this this");
byte[] b = doc.GetField("test2").BinaryValue();
Assert.AreEqual(b.Length, cmp.Length);
for (int j = 0; j < b.Length; j++)
Assert.AreEqual(b[j], cmp[j]);
}
}
finally
{
r.Close();
}
}
finally
{
dir.Close();
_TestUtil.RmDir(indexDir);
}
}
public virtual void runTest(Directory directory, bool autoCommit, MergeScheduler merger)
{
IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
writer.SetMaxBufferedDocs(2);
if (merger != null)
writer.SetMergeScheduler(merger);
for (int iter = 0; iter < NUM_ITER; iter++)
{
int iterFinal = iter;
writer.SetMergeFactor(1000);
for (int i = 0; i < 200; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d);
}
writer.SetMergeFactor(4);
//writer.setInfoStream(System.out);
int docCount = writer.DocCount();
SupportClass.ThreadClass[] threads = new SupportClass.ThreadClass[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
int iFinal = i;
IndexWriter writerFinal = writer;
threads[i] = new AnonymousClassThread(writerFinal, iFinal, iterFinal, this);
}
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Start();
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Join();
Assert.IsTrue(!failed);
int expectedDocCount = (int) ((1 + iter) * (200 + 8 * NUM_ITER2 * (NUM_THREADS / 2.0) * (1 + NUM_THREADS)));
// System.out.println("TEST: now index=" + writer.segString());
Assert.AreEqual(expectedDocCount, writer.DocCount());
if (!autoCommit)
{
writer.Close();
writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
writer.SetMaxBufferedDocs(2);
}
IndexReader reader = IndexReader.Open(directory);
Assert.IsTrue(reader.IsOptimized());
Assert.AreEqual(expectedDocCount, reader.NumDocs());
reader.Close();
}
writer.Close();
}
public virtual void TestTermVectorCorruption2()
{
Directory dir = new MockRAMDirectory();
for (int iter = 0; iter < 4; iter++)
{
bool autoCommit = 1 == iter / 2;
IndexWriter writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy());
Document document = new Document();
Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
document.Add(storedField);
writer.AddDocument(document);
writer.AddDocument(document);
document = new Document();
document.Add(storedField);
Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.UN_TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
document.Add(termVectorField);
writer.AddDocument(document);
writer.Optimize();
writer.Close();
IndexReader reader = IndexReader.Open(dir);
Assert.IsTrue(reader.GetTermFreqVectors(0) == null);
Assert.IsTrue(reader.GetTermFreqVectors(1) == null);
Assert.IsTrue(reader.GetTermFreqVectors(2) != null);
reader.Close();
}
dir.Close();
}
private void CreateIndexWriter()
{
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
indexWriter = new IndexWriter(directory, stopAnalyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
using (indexWriter.MergeScheduler) { }
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
// RavenDB already manages the memory for those, no need for Lucene to do this as well
indexWriter.SetMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
indexWriter.SetRAMBufferSizeMB(1024);
}
public virtual void TestNoExtraFiles()
{
RAMDirectory directory = new MockRAMDirectory();
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = pass == 0;
IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
for (int iter = 0; iter < 7; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
for (int j = 0; j < 21; j++)
{
Document doc = new Document();
doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles autoCommit=" + autoCommit);
// Reopen
writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
}
writer.Close();
}
directory.Close();
}
public virtual void TestNoExtraFiles()
{
RAMDirectory directory = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED);
for (int iter = 0; iter < 7; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
for (int j = 0; j < 21; j++)
{
Document doc = new Document();
doc.Add(new Field("content", "a b c", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
writer.Close();
TestIndexWriter.AssertNoUnreferencedFiles(directory, "testNoExtraFiles");
// Reopen
writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED);
}
writer.Close();
directory.Close();
}
public virtual void TestSetMaxMergeDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter iw = new IndexWriter(dir, new StandardAnalyzer(), true);
iw.SetMergeScheduler(new MyMergeScheduler(this));
iw.SetMaxMergeDocs(20);
iw.SetMaxBufferedDocs(2);
iw.SetMergeFactor(2);
Document document = new Document();
document.Add(new Field("tvtest", "a b c", Field.Store.NO, Field.Index.TOKENIZED, Field.TermVector.YES));
for (int i = 0; i < 177; i++)
iw.AddDocument(document);
iw.Close();
}
public virtual void runTest(Directory directory, MergeScheduler merger)
{
IndexWriter writer = new IndexWriter(directory, ANALYZER, true, IndexWriter.MaxFieldLength.UNLIMITED, null);
writer.SetMaxBufferedDocs(2);
if (merger != null)
{
writer.SetMergeScheduler(merger, null);
}
for (int iter = 0; iter < NUM_ITER; iter++)
{
int iterFinal = iter;
writer.MergeFactor = 1000;
for (int i = 0; i < 200; i++)
{
Document d = new Document();
d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
d.Add(new Field("contents", English.IntToEnglish(i), Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d, null);
}
writer.MergeFactor = 4;
//writer.setInfoStream(System.out);
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
int iFinal = i;
IndexWriter writerFinal = writer;
threads[i] = new AnonymousClassThread(writerFinal, iFinal, iterFinal, this);
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
Assert.IsTrue(!failed);
int expectedDocCount = (int)((1 + iter) * (200 + 8 * NUM_ITER2 * (NUM_THREADS / 2.0) * (1 + NUM_THREADS)));
// System.out.println("TEST: now index=" + writer.segString());
Assert.AreEqual(expectedDocCount, writer.MaxDoc());
writer.Close();
writer = new IndexWriter(directory, ANALYZER, false, IndexWriter.MaxFieldLength.UNLIMITED, null);
writer.SetMaxBufferedDocs(2);
IndexReader reader = IndexReader.Open(directory, true, null);
Assert.IsTrue(reader.IsOptimized());
Assert.AreEqual(expectedDocCount, reader.NumDocs());
reader.Close();
}
writer.Close();
}
private void OpenIndexOnStartup()
{
analyzer = new LowerCaseKeywordAnalyzer();
snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
bool resetTried = false;
bool recoveryTried = false;
while (true)
{
LuceneDirectory luceneDirectory = null;
try
{
luceneDirectory = OpenOrCreateLuceneDirectory(indexDirectory);
// Skip sanity test if we are running in memory. Index will not exist anyways.
if (!configuration.RunInMemory && !IsIndexStateValid(luceneDirectory))
throw new InvalidOperationException("Sanity check on the index failed.");
directory = luceneDirectory;
writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED);
writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true));
break;
}
catch (Exception e)
{
if (resetTried)
throw new InvalidOperationException("Could not open / create index for file system '" + name + "', reset already tried", e);
if (recoveryTried == false && luceneDirectory != null)
{
recoveryTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Trying to recover index", e);
StartupLog.Info("Recover functionality is still not implemented. Skipping.");
}
else
{
resetTried = true;
StartupLog.WarnException("Could not open index for file system '" + name + "'. Recovery operation failed, forcibly resetting index", e);
TryResettingIndex();
}
}
}
}
/*
Run one indexer and 2 searchers against single index as
stress test.
*/
public virtual void RunStressTest(Directory directory, bool autoCommit, MergeScheduler mergeScheduler)
{
IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true);
modifier.SetMaxBufferedDocs(10);
TimedThread[] threads = new TimedThread[4];
int numThread = 0;
if (mergeScheduler != null)
modifier.SetMergeScheduler(mergeScheduler);
// One modifier that writes 10 docs then removes 5, over
// and over:
IndexerThread indexerThread = new IndexerThread(this, modifier, threads);
threads[numThread++] = indexerThread;
indexerThread.Start();
IndexerThread indexerThread2 = new IndexerThread(this, modifier, threads);
threads[numThread++] = indexerThread2;
indexerThread2.Start();
// Two searchers that constantly just re-instantiate the
// searcher:
SearcherThread searcherThread1 = new SearcherThread(directory, threads);
threads[numThread++] = searcherThread1;
searcherThread1.Start();
SearcherThread searcherThread2 = new SearcherThread(directory, threads);
threads[numThread++] = searcherThread2;
searcherThread2.Start();
for (int i = 0; i < numThread; i++)
threads[i].Join();
modifier.Close();
for (int i = 0; i < numThread; i++)
Assert.IsTrue(!((TimedThread) threads[i]).failed);
//System.out.println(" Writer: " + indexerThread.count + " iterations");
//System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
//System.out.println("Searcher 2: " + searcherThread2.count + " searchers created");
}
public virtual void TestHangOnClose()
{
Directory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergePolicy(new LogByteSizeMergePolicy(writer));
writer.SetMaxBufferedDocs(5);
writer.UseCompoundFile = false;
writer.MergeFactor = 100;
Document doc = new Document();
doc.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 60; i++)
writer.AddDocument(doc);
writer.SetMaxBufferedDocs(200);
Document doc2 = new Document();
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
doc2.Add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, Field.Index.NO));
for (int i = 0; i < 10; i++)
writer.AddDocument(doc2);
writer.Close();
Directory dir2 = new MockRAMDirectory();
writer = new IndexWriter(dir2, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(writer);
lmp.MinMergeMB = 0.0001;
writer.SetMergePolicy(lmp);
writer.MergeFactor = 4;
writer.UseCompoundFile = false;
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.AddIndexesNoOptimize(new Directory[]{dir});
writer.Close();
dir.Close();
dir2.Close();
}
private void CreateIndexWriter()
{
indexWriter = new IndexWriter(directory, analyzer, indexDeletionPolicy, maxFieldLength);
if(_indexReaderWarmer!=null)
{
indexWriter.MergedSegmentWarmer = _indexReaderWarmer;
}
using (indexWriter.MergeScheduler) { }
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
// RavenDB already manages the memory for those, no need for Lucene to do this as well
indexWriter.SetMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
indexWriter.SetRAMBufferSizeMB(1024);
currentNumberOfWrites = 0;
}
public virtual void TestTermVectorCorruption()
{
Directory dir = new MockRAMDirectory();
for (int iter = 0; iter < 4; iter++)
{
bool autoCommit = 1 == iter / 2;
IndexWriter writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy(writer));
Document document = new Document();
Field storedField = new Field("stored", "stored", Field.Store.YES, Field.Index.NO);
document.Add(storedField);
writer.AddDocument(document);
writer.AddDocument(document);
document = new Document();
document.Add(storedField);
Field termVectorField = new Field("termVector", "termVector", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
document.Add(termVectorField);
writer.AddDocument(document);
writer.Optimize();
writer.Close();
IndexReader reader = IndexReader.Open(dir);
for (int i = 0; i < reader.NumDocs(); i++)
{
reader.Document(i);
reader.GetTermFreqVectors(i);
}
reader.Close();
writer = new IndexWriter(dir, autoCommit, new StandardAnalyzer());
writer.SetMaxBufferedDocs(2);
writer.SetRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
writer.SetMergeScheduler(new SerialMergeScheduler());
writer.SetMergePolicy(new LogDocMergePolicy(writer));
Directory[] indexDirs = new Directory[]{new MockRAMDirectory(dir)};
writer.AddIndexes(indexDirs);
writer.Close();
}
dir.Close();
}
public virtual void TestCloseWithThreads()
{
int NUM_THREADS = 3;
for (int iter = 0; iter < 50; iter++)
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(4);
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
//bool diskFull = false;
for (int i = 0; i < NUM_THREADS; i++)
threads[i] = new IndexerThread(this, writer, false);
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Start();
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 50));
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
}
writer.Close(false);
// Make sure threads that are adding docs are not hung:
for (int i = 0; i < NUM_THREADS; i++)
{
while (true)
{
try
{
// Without fix for LUCENE-1130: one of the
// threads will hang
threads[i].Join();
break;
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
}
}
if (threads[i].IsAlive)
Assert.Fail("thread seems to be hung");
}
// Quick test to make sure index is not corrupt:
IndexReader reader = IndexReader.Open(dir);
TermDocs tdocs = reader.TermDocs(new Term("field", "aaa"));
int count = 0;
while (tdocs.Next())
{
count++;
}
Assert.IsTrue(count > 0);
reader.Close();
dir.Close();
}
}
public virtual void TestExceptionDuringSync()
{
MockRAMDirectory dir = new MockRAMDirectory();
FailOnlyInSync failure = new FailOnlyInSync();
dir.FailOn(failure);
IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer());
failure.SetDoFail();
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
// We expect sync exceptions in the merge threads
cms.SetSuppressExceptions();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(5);
for (int i = 0; i < 23; i++)
AddDoc(writer);
cms.Sync();
Assert.IsTrue(failure.didFail);
failure.ClearDoFail();
writer.Close();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(23, reader.NumDocs());
reader.Close();
dir.Close();
}
public virtual void TestImmediateDiskFullWithThreads()
{
int NUM_THREADS = 3;
for (int iter = 0; iter < 10; iter++)
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
// We expect disk full exceptions in the merge threads
cms.SetSuppressExceptions_ForNUnitTest();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(4);
dir.SetMaxSizeInBytes(4 * 1024 + 20 * iter);
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
//bool diskFull = false;
for (int i = 0; i < NUM_THREADS; i++)
threads[i] = new IndexerThread(this, writer, true);
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Start();
for (int i = 0; i < NUM_THREADS; i++)
{
while (true)
{
try
{
// Without fix for LUCENE-1130: one of the
// threads will hang
threads[i].Join();
break;
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
}
}
if (threads[i].IsAlive)
Assert.Fail("thread seems to be hung");
else
Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable");
}
try
{
writer.Close(false);
}
catch (System.IO.IOException)
{
}
dir.Close();
}
}
public virtual void TestDeleteMerging()
{
RAMDirectory directory = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(directory, true, ANALYZER, true);
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
LogDocMergePolicy mp = new LogDocMergePolicy(writer);
writer.SetMergePolicy(mp);
// Force degenerate merging so we can get a mix of
// merging of segments with and without deletes at the
// start:
mp.SetMinMergeDocs(1000);
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int i = 0; i < 10; i++)
{
for (int j = 0; j < 100; j++)
{
idField.SetValue(System.Convert.ToString(i * 100 + j));
writer.AddDocument(doc);
}
int delID = i;
while (delID < 100 * (1 + i))
{
writer.DeleteDocuments(new Term("id", "" + delID));
delID += 10;
}
writer.Flush();
}
writer.Close();
IndexReader reader = IndexReader.Open(directory);
// Verify that we did not lose any deletes...
Assert.AreEqual(450, reader.NumDocs());
reader.Close();
directory.Close();
}
// Runs test, with multiple threads, using the specific
// failure to trigger an IOException
public virtual void _testMultipleThreadsFailure(MockRAMDirectory.Failure failure)
{
int NUM_THREADS = 3;
for (int iter = 0; iter < 5; iter++)
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
// We expect disk full exceptions in the merge threads
cms.SetSuppressExceptions_ForNUnitTest();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(4);
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
//bool diskFull = false;
for (int i = 0; i < NUM_THREADS; i++)
threads[i] = new IndexerThread(this, writer, true);
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Start();
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 10));
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
}
dir.FailOn(failure);
failure.SetDoFail();
for (int i = 0; i < NUM_THREADS; i++)
{
while (true)
{
try
{
threads[i].Join();
break;
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
}
}
if (threads[i].IsAlive)
Assert.Fail("thread seems to be hung");
else
Assert.IsTrue(threads[i].error == null, "hit unexpected Throwable");
}
bool success = false;
try
{
writer.Close(false);
success = true;
}
catch (System.IO.IOException)
{
}
if (success)
{
IndexReader reader = IndexReader.Open(dir);
for (int j = 0; j < reader.MaxDoc(); j++)
{
if (!reader.IsDeleted(j))
{
reader.Document(j);
reader.GetTermFreqVectors(j);
}
}
reader.Close();
}
dir.Close();
}
}
public virtual void TestNoWaitClose()
{
RAMDirectory directory = new MockRAMDirectory();
Document doc = new Document();
Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.Add(idField);
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = pass == 0;
IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
for (int iter = 0; iter < 10; iter++)
{
ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
writer.SetMergeScheduler(cms);
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(100);
for (int j = 0; j < 201; j++)
{
idField.SetValue(System.Convert.ToString(iter * 201 + j));
writer.AddDocument(doc);
}
int delID = iter * 201;
for (int j = 0; j < 20; j++)
{
writer.DeleteDocuments(new Term("id", System.Convert.ToString(delID)));
delID += 5;
}
// Force a bunch of merge threads to kick off so we
// stress out aborting them on close:
writer.SetMergeFactor(3);
writer.AddDocument(doc);
writer.Flush();
writer.Close(false);
IndexReader reader = IndexReader.Open(directory);
Assert.AreEqual((1 + iter) * 182, reader.NumDocs());
reader.Close();
// Reopen
writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
}
writer.Close();
}
directory.Close();
}
/*
Run one indexer and 2 searchers against single index as
stress test.
*/
public virtual void RunStressTest(Directory directory, bool autoCommit, MergeScheduler mergeScheduler)
{
IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true);
modifier.SetMaxBufferedDocs(10);
TimedThread[] threads = new TimedThread[4];
if (mergeScheduler != null)
modifier.SetMergeScheduler(mergeScheduler);
// One modifier that writes 10 docs then removes 5, over
// and over:
IndexerThread indexerThread = new IndexerThread(modifier, threads);
threads[0] = indexerThread;
indexerThread.Start();
IndexerThread indexerThread2 = new IndexerThread(modifier, threads);
threads[2] = indexerThread2;
indexerThread2.Start();
// Two searchers that constantly just re-instantiate the
// searcher:
SearcherThread searcherThread1 = new SearcherThread(directory, threads);
threads[3] = searcherThread1;
searcherThread1.Start();
SearcherThread searcherThread2 = new SearcherThread(directory, threads);
threads[3] = searcherThread2;
searcherThread2.Start();
indexerThread.Join();
indexerThread2.Join();
searcherThread1.Join();
searcherThread2.Join();
modifier.Close();
Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer");
Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2");
Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1");
Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2");
//System.out.println(" Writer: " + indexerThread.count + " iterations");
//System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
//System.out.println("Searcher 2: " + searcherThread2.count + " searchers created");
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 4; pass++)
{
bool autoCommit = pass < 2;
bool useCompoundFile = (pass % 2) > 0;
// Never deletes a commit
KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(this);
Directory dir = new RAMDirectory();
policy.dir = dir;
IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
writer.SetMaxBufferedDocs(10);
writer.SetUseCompoundFile(useCompoundFile);
writer.SetMergeScheduler(new SerialMergeScheduler());
for (int i = 0; i < 107; i++)
{
AddDoc(writer);
if (autoCommit && i % 10 == 0)
{
writer.Commit();
}
}
writer.Close();
writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
writer.SetUseCompoundFile(useCompoundFile);
writer.Optimize();
writer.Close();
Assert.AreEqual(2, policy.numOnInit);
if (!autoCommit)
{
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(2, policy.numOnCommit);
}
// Test listCommits
System.Collections.ICollection commits = IndexReader.ListCommits(dir);
if (!autoCommit)
{
// 1 from opening writer + 2 from closing writer
Assert.AreEqual(3, commits.Count);
}
// 1 from opening writer + 2 from closing writer +
// 11 from calling writer.commit() explicitly above
else
{
Assert.AreEqual(14, commits.Count);
}
System.Collections.IEnumerator it = commits.GetEnumerator();
// Make sure we can open a reader on each commit:
while (it.MoveNext())
{
IndexCommit commit = (IndexCommit)it.Current;
IndexReader r = IndexReader.Open(commit, null);
r.Close();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetCurrentSegmentGeneration(dir);
while (gen > 0)
{
IndexReader reader = IndexReader.Open(dir);
reader.Close();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.ListAll().Length;
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, policy, IndexWriter.MaxFieldLength.LIMITED);
writer.Close();
int postCount = dir.ListAll().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Close();
}
}
/// <summary> Close the IndexReader and open an IndexWriter.</summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if there is a low-level IO error </throws>
protected internal virtual void CreateIndexWriter()
{
if (indexWriter == null)
{
if (indexReader != null)
{
indexReader.Close();
indexReader = null;
}
indexWriter = new IndexWriter(directory, analyzer, false, new IndexWriter.MaxFieldLength(maxFieldLength));
// IndexModifier cannot use ConcurrentMergeScheduler
// because it synchronizes on the directory which can
// cause deadlock
indexWriter.SetMergeScheduler(new SerialMergeScheduler());
indexWriter.SetInfoStream(infoStream);
indexWriter.SetUseCompoundFile(useCompoundFile);
if (maxBufferedDocs != IndexWriter.DISABLE_AUTO_FLUSH)
indexWriter.SetMaxBufferedDocs(maxBufferedDocs);
indexWriter.SetMergeFactor(mergeFactor);
}
}
private static IndexWriter CreateIndexWriter(Directory directory)
{
var indexWriter = new IndexWriter(directory, new StopAnalyzer(Version.LUCENE_29), IndexWriter.MaxFieldLength.UNLIMITED);
var mergeScheduler = indexWriter.GetMergeScheduler();
if(mergeScheduler != null)
mergeScheduler.Close();
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
return indexWriter;
}
private void DoTestReopenWithCommit(Directory dir, bool withReopen)
{
IndexWriter iwriter = new IndexWriter(dir, new KeywordAnalyzer(), true, MaxFieldLength.LIMITED);
iwriter.SetMergeScheduler(new SerialMergeScheduler());
IndexReader reader = IndexReader.Open(dir);
try
{
int M = 3;
for (int i = 0; i < 4; i++)
{
for (int j = 0; j < M; j++)
{
Document doc = new Document();
doc.Add(new Field("id", i + "_" + j, Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("id2", i + "_" + j, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
doc.Add(new Field("id3", i + "_" + j, Field.Store.YES, Field.Index.NO));
iwriter.AddDocument(doc);
if (i > 0)
{
int k = i - 1;
int n = j + k * M;
Document prevItereationDoc = reader.Document(n);
Assert.IsNotNull(prevItereationDoc);
System.String id = prevItereationDoc.Get("id");
Assert.AreEqual(k + "_" + j, id);
}
}
iwriter.Commit();
if (withReopen)
{
// reopen
IndexReader r2 = reader.Reopen();
if (reader != r2)
{
reader.Close();
reader = r2;
}
}
else
{
// recreate
reader.Close();
reader = IndexReader.Open(dir);
}
}
}
finally
{
iwriter.Close();
reader.Close();
}
}
internal void TryResettingIndex()
{
try
{
IOExtensions.DeleteDirectory(indexDirectory);
using ( LuceneDirectory luceneDirectory = FSDirectory.Open(new DirectoryInfo(indexDirectory)) )
{
WriteIndexVersion(luceneDirectory);
using (var indexWriter = new IndexWriter(luceneDirectory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED))
{
indexWriter.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler());
filesystem.Storage.Batch(accessor =>
{
foreach (var file in accessor.GetFilesAfter(Etag.Empty, int.MaxValue))
{
Index(indexWriter, FileHeader.Canonize(file.FullPath), file.Metadata, file.Etag, recreateSearcher: false);
}
});
indexWriter.Flush(true, true, true);
}
}
}
catch (Exception exception)
{
throw new InvalidOperationException("Could not reset index for file system: " + name, exception);
}
}
public virtual void TestBackgroundOptimize()
{
Directory dir = new MockRAMDirectory();
for (int pass = 0; pass < 2; pass++)
{
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
writer.SetMergeScheduler(new ConcurrentMergeScheduler());
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.SetMaxBufferedDocs(2);
writer.SetMergeFactor(101);
for (int i = 0; i < 200; i++)
writer.AddDocument(doc);
writer.Optimize(false);
if (0 == pass)
{
writer.Close();
IndexReader reader = IndexReader.Open(dir);
Assert.IsTrue(reader.IsOptimized());
reader.Close();
}
else
{
// Get another segment to flush so we can verify it is
// NOT included in the optimization
writer.AddDocument(doc);
writer.AddDocument(doc);
writer.Close();
IndexReader reader = IndexReader.Open(dir);
Assert.IsTrue(!reader.IsOptimized());
reader.Close();
SegmentInfos infos = new SegmentInfos();
infos.Read(dir);
Assert.AreEqual(2, infos.Count);
}
}
dir.Close();
}