public void TestSubclassConcurrentMergeScheduler()
{
MockDirectoryWrapper dir = NewMockDirectory();
dir.FailOn(new FailOnlyOnMerge());
Document doc = new Document();
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(new MyMergeScheduler(this)).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy()));
LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy;
logMP.MergeFactor = 10;
for (int i = 0; i < 20; i++)
{
writer.AddDocument(doc);
}
((MyMergeScheduler)writer.Config.MergeScheduler).Sync();
writer.Dispose();
Assert.IsTrue(MergeThreadCreated);
Assert.IsTrue(MergeCalled);
Assert.IsTrue(ExcCalled);
dir.Dispose();
}
// Runs test, with one thread, using the specific failure
// to trigger an IOException
public virtual void TestSingleThreadFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure)
{
MockDirectoryWrapper dir = NewMockDirectory();
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2).SetMergeScheduler(newScheduler()));
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
for (int i = 0; i < 6; i++)
{
writer.AddDocument(doc);
}
dir.FailOn(failure);
failure.SetDoFail();
try
{
writer.AddDocument(doc);
writer.AddDocument(doc);
writer.Commit();
Assert.Fail("did not hit exception");
}
catch (IOException)
{
}
failure.ClearDoFail();
writer.AddDocument(doc);
writer.Dispose(false);
dir.Dispose();
}
public void TestExceptionOnBackgroundThreadIsPropagatedToCallingThread()
{
using MockDirectoryWrapper dir = NewMockDirectory();
dir.FailOn(new FailOnlyOnMerge());
Document doc = new Document();
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
var mergeScheduler = new ConcurrentMergeScheduler();
using IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(mergeScheduler).SetMaxBufferedDocs(2).SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).SetMergePolicy(NewLogMergePolicy()));
LogMergePolicy logMP = (LogMergePolicy)writer.Config.MergePolicy;
logMP.MergeFactor = 10;
for (int i = 0; i < 20; i++)
{
writer.AddDocument(doc);
}
bool exceptionHit = false;
try
{
mergeScheduler.Sync();
}
catch (MergePolicy.MergeException)
{
exceptionHit = true;
}
assertTrue(exceptionHit);
}
public virtual void TestExceptionDuringSave()
{
MockDirectoryWrapper dir = NewMockDirectory();
dir.FailOn(new FailureAnonymousInnerClassHelper(this, dir));
IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND)));
writer.AddDocument(new Document());
writer.Commit();
PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
try
{
psdp.Snapshot();
}
catch (IOException ioe)
{
if (ioe.Message.Equals("now fail on purpose", StringComparison.Ordinal))
{
// ok
}
else
{
throw ioe;
}
}
Assert.AreEqual(0, psdp.SnapshotCount);
writer.Dispose();
Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
dir.Dispose();
}
public virtual void TestExceptionDuringSave()
{
MockDirectoryWrapper dir = NewMockDirectory();
dir.FailOn(new FailureAnonymousClass(this, dir));
IndexWriter writer = new IndexWriter(dir, GetConfig(Random, new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND)));
writer.AddDocument(new Document());
writer.Commit();
PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
try
{
psdp.Snapshot();
}
catch (Exception ioe) when(ioe.IsIOException())
{
if (ioe.Message.Equals("now fail on purpose", StringComparison.Ordinal))
{
// ok
}
else
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
}
Assert.AreEqual(0, psdp.SnapshotCount);
writer.Dispose();
Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
dir.Dispose();
}
public virtual void TestTransactions_Mem(
[ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler1,
[ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler2)
{
Console.WriteLine("Start test");
// we cant use non-ramdir on windows, because this test needs to double-write.
MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory());
MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory());
dir1.PreventDoubleWrite = false;
dir2.PreventDoubleWrite = false;
dir1.FailOn(new RandomFailure(this));
dir2.FailOn(new RandomFailure(this));
dir1.FailOnOpenInput = false;
dir2.FailOnOpenInput = false;
// We throw exceptions in deleteFile, which creates
// leftover files:
dir1.AssertNoUnrefencedFilesOnClose = false;
dir2.AssertNoUnrefencedFilesOnClose = false;
InitIndex(dir1);
InitIndex(dir2);
TimedThread[] threads = new TimedThread[3];
int numThread = 0;
IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, newScheduler1, newScheduler2, threads);
threads[numThread++] = indexerThread;
indexerThread.Start();
SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads);
threads[numThread++] = searcherThread1;
searcherThread1.Start();
SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads);
threads[numThread++] = searcherThread2;
searcherThread2.Start();
for (int i = 0; i < numThread; i++)
{
threads[i].Join();
}
for (int i = 0; i < numThread; i++)
{
Assert.IsTrue(!threads[i].Failed);
}
dir1.Dispose();
dir2.Dispose();
Console.WriteLine("End test");
}
public virtual void TestCorruptionAfterDiskFullDuringMerge()
{
MockDirectoryWrapper dir = NewMockDirectory();
//IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderPooling(true));
IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(new SerialMergeScheduler()).SetReaderPooling(true).SetMergePolicy(NewLogMergePolicy(2)));
// we can do this because we add/delete/add (and dont merge to "nothing")
w.KeepFullyDeletedSegments = true;
Document doc = new Document();
doc.Add(NewTextField("f", "doctor who", Field.Store.NO));
w.AddDocument(doc);
w.Commit();
w.DeleteDocuments(new Term("f", "who"));
w.AddDocument(doc);
// disk fills up!
FailTwiceDuringMerge ftdm = new FailTwiceDuringMerge();
ftdm.SetDoFail();
dir.FailOn(ftdm);
try
{
w.Commit();
Assert.Fail("fake disk full IOExceptions not hit");
}
#pragma warning disable 168
catch (IOException ioe)
#pragma warning restore 168
{
// expected
Assert.IsTrue(ftdm.DidFail1 || ftdm.DidFail2);
}
TestUtil.CheckIndex(dir);
ftdm.ClearDoFail();
w.AddDocument(doc);
w.Dispose();
dir.Dispose();
}
public virtual void TestFlushExceptions()
{
MockDirectoryWrapper directory = NewMockDirectory();
FailOnlyOnFlush failure = new FailOnlyOnFlush(this);
directory.FailOn(failure);
IndexWriter writer = new IndexWriter(directory, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
Document doc = new Document();
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
int extraCount = 0;
for (int i = 0; i < 10; i++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + i);
}
for (int j = 0; j < 20; j++)
{
idField.StringValue = Convert.ToString(i * 20 + j);
writer.AddDocument(doc);
}
// must cycle here because sometimes the merge flushes
// the doc we just added and so there's nothing to
// flush, and we don't hit the exception
while (true)
{
writer.AddDocument(doc);
failure.SetDoFail();
try
{
writer.Flush(true, true);
if (failure.HitExc)
{
Assert.Fail("failed to hit IOException");
}
extraCount++;
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine(ioe.StackTrace);
}
failure.ClearDoFail();
break;
}
}
Assert.AreEqual(20 * (i + 1) + extraCount, writer.NumDocs());
}
writer.Dispose();
IndexReader reader = DirectoryReader.Open(directory);
Assert.AreEqual(200 + extraCount, reader.NumDocs);
reader.Dispose();
directory.Dispose();
}
// Runs test, with multiple threads, using the specific
// failure to trigger an IOException
public virtual void TestMultipleThreadsFailure(Func <IConcurrentMergeScheduler> newScheduler, Failure failure)
{
int NUM_THREADS = 3;
for (int iter = 0; iter < 2; iter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
MockDirectoryWrapper dir = NewMockDirectory();
var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(2)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(4));
IndexWriter writer = new IndexWriter(dir, config);
var scheduler = config.mergeScheduler as IConcurrentMergeScheduler;
if (scheduler != null)
{
scheduler.SetSuppressExceptions();
}
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(writer, true, NewField);
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Start();
}
Thread.Sleep(10);
dir.FailOn(failure);
failure.SetDoFail();
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
Assert.IsTrue(threads[i].Error == null, "hit unexpected Throwable");
}
bool success = false;
try
{
writer.Dispose(false);
success = true;
}
catch (IOException)
{
failure.ClearDoFail();
writer.Dispose(false);
}
if (VERBOSE)
{
Console.WriteLine("TEST: success=" + success);
}
if (success)
{
IndexReader reader = DirectoryReader.Open(dir);
IBits delDocs = MultiFields.GetLiveDocs(reader);
for (int j = 0; j < reader.MaxDoc; j++)
{
if (delDocs == null || !delDocs.Get(j))
{
reader.Document(j);
reader.GetTermVectors(j);
}
}
reader.Dispose();
}
dir.Dispose();
}
}
public virtual void TestTransactions_Mem()
{
// we cant use non-ramdir on windows, because this test needs to double-write.
MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory());
MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory());
dir1.PreventDoubleWrite = false;
dir2.PreventDoubleWrite = false;
dir1.FailOn(new RandomFailure(this));
dir2.FailOn(new RandomFailure(this));
dir1.FailOnOpenInput = false;
dir2.FailOnOpenInput = false;
// We throw exceptions in deleteFile, which creates
// leftover files:
dir1.AssertNoUnrefencedFilesOnClose = false;
dir2.AssertNoUnrefencedFilesOnClose = false;
InitIndex(dir1);
InitIndex(dir2);
TimedThread[] threads = new TimedThread[3];
int numThread = 0;
IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, threads);
threads[numThread++] = indexerThread;
indexerThread.Start();
SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads);
threads[numThread++] = searcherThread1;
searcherThread1.Start();
SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads);
threads[numThread++] = searcherThread2;
searcherThread2.Start();
for (int i = 0; i < numThread; i++)
{
threads[i].Join();
}
for (int i = 0; i < numThread; i++)
{
Assert.IsTrue(!threads[i].Failed);
}
dir1.Dispose();
dir2.Dispose();
}