public override void Run()
{
try
{
while (!Stop)
{
Ctrl.WaitIfStalled();
if (CheckPoint)
{
#if !NETSTANDARD1_6
try
{
#endif
Assert.IsTrue(Sync.await());
#if !NETSTANDARD1_6
}
catch (ThreadInterruptedException /*e*/)
{
Console.WriteLine("[Waiter] got interrupted - wait count: " + Sync.Waiter.CurrentCount);
//throw new ThreadInterruptedException("Thread Interrupted Exception", e);
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
#endif
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Exceptions.Add(e);
}
}
public virtual void Test()
{
IndexReader reader = null;
try
{
reader = DirectoryReader.Open(Directory);
for (int i = 1; i <= NumThreads; i++)
{
TestTermPositionVectors(reader, i);
}
}
catch (IOException ioe)
{
Assert.Fail(ioe.Message);
}
finally
{
if (reader != null)
{
try
{
/// <summary>
/// close the opened reader </summary>
reader.Dispose();
}
catch (IOException ioe)
{
Console.WriteLine(ioe.ToString());
Console.Write(ioe.StackTrace);
}
}
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
Console.WriteLine("NRT reopen times:");
for (int i = 0; i < reopenCount; i++)
{
Console.Write(" " + reopenTimes[i]);
}
Console.WriteLine();
}
}
public override void Run()
{
try
{
while (!Stop)
{
int internalIters = Release && Random.NextBoolean() ? AtLeast(5) : 1;
for (int i = 0; i < internalIters; i++)
{
Ctrl.UpdateStalled(Random.NextBoolean());
}
if (CheckPoint)
{
Sync.UpdateJoin.Signal();
try
{
Assert.IsTrue(Sync.await());
}
#if !NETSTANDARD1_6
catch (ThreadInterruptedException /*e*/)
{
Console.WriteLine("[Updater] got interrupted - wait count: " + Sync.Waiter.CurrentCount);
//throw new ThreadInterruptedException("Thread Interrupted Exception", e);
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
#endif
catch (Exception e)
{
Console.Write("signal failed with : " + e);
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
Sync.LeftCheckpoint.Signal();
}
if (Random.NextBoolean())
{
Thread.Sleep(0);
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Exceptions.Add(e);
}
if (!Sync.UpdateJoin.IsSet)
{
Sync.UpdateJoin.Signal();
}
}
public virtual void TestLaaaaaargeOutput()
{
string message = "I will not OOM on large output";
int howMuch = 250 * 1024 * 1024;
for (int i = 0; i < howMuch; i++)
{
if (i > 0)
{
Console.Write(",\n");
}
Console.Write(message);
howMuch -= message.Length; // approximately.
}
Console.WriteLine(".");
}
public void Run()
{
try
{
// run the test 100 times
for (int i = 0; i < RunsToDo; i++)
{
TestTermVectors();
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
}
return;
}
public override void Run()
{
try
{
for (int j = 0; j < NumDocs; j++)
{
Document d = new Document();
d.Add(new TextField(Field, new PoolingPayloadTokenStream(OuterInstance, Pool)));
Writer.AddDocument(d);
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(e.ToString());
}
}
public override void Run()
{
try
{
signal.Wait();
manager.MaybeRefresh();
writer.DeleteDocuments(new TermQuery(new Term("foo", "barista")));
manager.MaybeRefresh(); // kick off another reopen so we inc. the internal gen
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
}
finally
{
latch.Reset(latch.CurrentCount == 0 ? 0 : latch.CurrentCount - 1); // let the add below finish
}
}
public virtual void TestIsCurrentWithThreads()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
IndexWriter writer = new IndexWriter(dir, conf);
ReaderHolder holder = new ReaderHolder();
ReaderThread[] threads = new ReaderThread[AtLeast(3)];
CountdownEvent latch = new CountdownEvent(1);
WriterThread writerThread = new WriterThread(holder, writer, AtLeast(500), Random, latch);
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new ReaderThread(holder, latch);
threads[i].Start();
}
writerThread.Start();
writerThread.Join();
bool failed = writerThread.Failed != null;
if (failed)
{
Console.WriteLine(writerThread.Failed.ToString());
Console.Write(writerThread.Failed.StackTrace);
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
if (threads[i].Failed != null)
{
Console.WriteLine(threads[i].Failed.ToString());
Console.Write(threads[i].Failed.StackTrace);
failed = true;
}
}
Assert.IsFalse(failed);
writer.Dispose();
dir.Dispose();
}
public virtual void TestRandomMaps2()
{
Random random = Random;
int numIterations = AtLeast(3);
for (int iter = 0; iter < numIterations; iter++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST iter=" + iter);
}
char endLetter = (char)TestUtil.NextInt32(random, 'b', 'z');
IDictionary <string, string> map = new Dictionary <string, string>();
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
int numMappings = AtLeast(5);
if (VERBOSE)
{
Console.WriteLine(" mappings:");
}
while (map.Count < numMappings)
{
string key = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, 7);
if (key.Length != 0 && !map.ContainsKey(key))
{
string value = TestUtil.RandomSimpleString(random);
map[key] = value;
builder.Add(key, value);
if (VERBOSE)
{
Console.WriteLine(" " + key + " -> " + value);
}
}
}
NormalizeCharMap charMap = builder.Build();
if (VERBOSE)
{
Console.WriteLine(" test random documents...");
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
string content = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, AtLeast(1000));
if (VERBOSE)
{
Console.WriteLine(" content=" + content);
}
// Do stupid dog-slow mapping:
// Output string:
StringBuilder output = new StringBuilder();
// Maps output offset to input offset:
IList <int?> inputOffsets = new List <int?>();
int cumDiff = 0;
int charIdx = 0;
while (charIdx < content.Length)
{
int matchLen = -1;
string matchRepl = null;
foreach (KeyValuePair <string, string> ent in map)
{
string match = ent.Key;
if (charIdx + match.Length <= content.Length)
{
int limit = charIdx + match.Length;
bool matches = true;
for (int charIdx2 = charIdx; charIdx2 < limit; charIdx2++)
{
if (match[charIdx2 - charIdx] != content[charIdx2])
{
matches = false;
break;
}
}
if (matches)
{
string repl = ent.Value;
if (match.Length > matchLen)
{
// Greedy: longer match wins
matchLen = match.Length;
matchRepl = repl;
}
}
}
}
if (matchLen != -1)
{
// We found a match here!
if (VERBOSE)
{
Console.WriteLine(" match=" + content.Substring(charIdx, matchLen) + " @ off=" + charIdx + " repl=" + matchRepl);
}
output.Append(matchRepl);
int minLen = Math.Min(matchLen, matchRepl.Length);
// Common part, directly maps back to input
// offset:
for (int outIdx = 0; outIdx < minLen; outIdx++)
{
inputOffsets.Add(output.Length - matchRepl.Length + outIdx + cumDiff);
}
cumDiff += matchLen - matchRepl.Length;
charIdx += matchLen;
if (matchRepl.Length < matchLen)
{
// Replacement string is shorter than matched
// input: nothing to do
}
else if (matchRepl.Length > matchLen)
{
// Replacement string is longer than matched
// input: for all the "extra" chars we map
// back to a single input offset:
for (int outIdx = matchLen; outIdx < matchRepl.Length; outIdx++)
{
inputOffsets.Add(output.Length + cumDiff - 1);
}
}
else
{
// Same length: no change to offset
}
Debug.Assert(inputOffsets.Count == output.Length, "inputOffsets.size()=" + inputOffsets.Count + " vs output.length()=" + output.Length);
}
else
{
inputOffsets.Add(output.Length + cumDiff);
output.Append(content[charIdx]);
charIdx++;
}
}
string expected = output.ToString();
if (VERBOSE)
{
Console.Write(" expected:");
for (int charIdx2 = 0; charIdx2 < expected.Length; charIdx2++)
{
Console.Write(" " + expected[charIdx2] + "/" + inputOffsets[charIdx2]);
}
Console.WriteLine();
}
MappingCharFilter mapFilter = new MappingCharFilter(charMap, new StringReader(content));
StringBuilder actualBuilder = new StringBuilder();
IList <int?> actualInputOffsets = new List <int?>();
// Now consume the actual mapFilter, somewhat randomly:
while (true)
{
if (random.Next(0, 1) == 1)
{
int ch = mapFilter.Read();
if (ch == -1)
{
break;
}
actualBuilder.Append((char)ch);
}
else
{
char[] buffer = new char[TestUtil.NextInt32(random, 1, 100)];
int off = buffer.Length == 1 ? 0 : random.Next(buffer.Length - 1);
int count = mapFilter.Read(buffer, off, buffer.Length - off);
if (count == -1)
{
break;
}
else
{
actualBuilder.Append(buffer, off, count);
}
}
if (random.Next(10) == 7)
{
// Map offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
}
}
// Finish mappping offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
string actual = actualBuilder.ToString();
// Verify:
assertEquals(expected, actual);
assertEquals(inputOffsets, actualInputOffsets);
}
}
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadJob[numReleasers + numStallers + numWaiters];
IList <Exception> exceptions = new SynchronizedList <Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint)
{
Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked && ctrl.IsHealthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Value = (false);
sync.Waiter.Signal();
sync.LeftCheckpoint.Wait();
}
Assert.IsFalse(checkPoint);
Assert.AreEqual(0, sync.Waiter.CurrentCount);
if (checkPointProbability >= (float)Random.NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
}
if (!checkPoint)
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Value = (false);
stop.Value = (true);
sync.Waiter.Signal();
sync.LeftCheckpoint.Wait();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public virtual void Test()
{
MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors"));
dir.PreventDoubleWrite = false;
double rate = Random.NextDouble() * 0.01;
//System.out.println("rate=" + rate);
dir.RandomIOExceptionRateOnOpen = rate;
int iters = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
IndexReader r = null;
DirectoryReader r2 = null;
bool any = false;
MockDirectoryWrapper dirCopy = null;
int lastNumDocs = 0;
for (int iter = 0; iter < iters; iter++)
{
IndexWriter w = null;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
try
{
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
if (VERBOSE)
{
// Do this ourselves instead of relying on LTC so
// we see incrementing messageID:
iwc.SetInfoStream(new TextWriterInfoStream(Console.Out));
}
var ms = iwc.MergeScheduler;
if (ms is IConcurrentMergeScheduler)
{
((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
}
w = new IndexWriter(dir, iwc);
if (r != null && Random.Next(5) == 3)
{
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes IR[]");
}
w.AddIndexes(new IndexReader[] { r });
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes Directory[]");
}
w.AddIndexes(new Directory[] { dirCopy });
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addDocument");
}
w.AddDocument(docs.NextDoc());
}
dir.RandomIOExceptionRateOnOpen = 0.0;
w.Dispose();
w = null;
// NOTE: this is O(N^2)! Only enable for temporary debugging:
//dir.setRandomIOExceptionRateOnOpen(0.0);
//TestUtil.CheckIndex(dir);
//dir.setRandomIOExceptionRateOnOpen(rate);
// Verify numDocs only increases, to catch IndexWriter
// accidentally deleting the index:
dir.RandomIOExceptionRateOnOpen = 0.0;
Assert.IsTrue(DirectoryReader.IndexExists(dir));
if (r2 == null)
{
r2 = DirectoryReader.Open(dir);
}
else
{
DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2);
if (r3 != null)
{
r2.Dispose();
r2 = r3;
}
}
Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs);
lastNumDocs = r2.NumDocs;
//System.out.println("numDocs=" + lastNumDocs);
dir.RandomIOExceptionRateOnOpen = rate;
any = true;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": success");
}
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": exception");
Console.WriteLine(ioe.ToString());
Console.Write(ioe.StackTrace);
}
if (w != null)
{
// NOTE: leave random IO exceptions enabled here,
// to verify that rollback does not try to write
// anything:
w.Rollback();
}
}
if (any && r == null && Random.NextBoolean())
{
// Make a copy of a non-empty index so we can use
// it to addIndexes later:
dir.RandomIOExceptionRateOnOpen = 0.0;
r = DirectoryReader.Open(dir);
dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
ISet <string> files = new JCG.HashSet <string>();
foreach (string file in dir.ListAll())
{
dir.Copy(dirCopy, file, file, IOContext.DEFAULT);
files.Add(file);
}
dirCopy.Sync(files);
// Have IW kiss the dir so we remove any leftover
// files ... we can easily have leftover files at
// the time we take a copy because we are holding
// open a reader:
(new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose();
dirCopy.RandomIOExceptionRate = rate;
dir.RandomIOExceptionRateOnOpen = rate;
}
}
if (r2 != null)
{
r2.Dispose();
}
if (r != null)
{
r.Dispose();
dirCopy.Dispose();
}
dir.Dispose();
}
