public virtual void TestRandomExceptionsThreads()
{
random = new Random((int)(DateTime.Now.Ticks & 0x7fffffff));
MockRAMDirectory dir = new MockRAMDirectory();
MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
((ConcurrentMergeScheduler)writer.MergeScheduler).SetSuppressExceptions();
//writer.setMaxBufferedDocs(10);
writer.SetRAMBufferSizeMB(0.2);
if (DEBUG)
{
System.IO.StreamWriter temp_writer;
temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
temp_writer.AutoFlush = true;
writer.SetInfoStream(temp_writer, null);
}
int NUM_THREADS = 4;
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(this, i, writer);
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i].Join();
}
for (int i = 0; i < NUM_THREADS; i++)
{
Assert.IsNull(threads[i].failure, "thread " + threads[i].Name + ": hit unexpected failure");
}
writer.Commit(null);
try
{
writer.Close();
}
catch (System.Exception t)
{
System.Console.Out.WriteLine("exception during close:");
System.Console.Out.WriteLine(t.StackTrace);
writer.Rollback(null);
}
// Confirm that when doc hits exception partway through tokenization, it's deleted:
IndexReader r2 = IndexReader.Open((Directory)dir, true, null);
int count = r2.DocFreq(new Term("content4", "aaa"), null);
int count2 = r2.DocFreq(new Term("content4", "ddd"), null);
Assert.AreEqual(count, count2);
r2.Close();
_TestUtil.CheckIndex(dir);
}
public virtual DocsAndWriter IndexRandomIWReader(int nThreads, int iterations, int range, Directory dir)
{
System.Collections.Hashtable docs = new System.Collections.Hashtable();
IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true);
w.SetUseCompoundFile(false);
/***
* w.setMaxMergeDocs(Integer.MAX_VALUE);
* w.setMaxFieldLength(10000);
* w.setRAMBufferSizeMB(1);
* w.setMergeFactor(10);
***/
// force many merges
w.SetMergeFactor(mergeFactor);
w.SetRAMBufferSizeMB(.1);
w.SetMaxBufferedDocs(maxBufferedDocs);
threads = new IndexingThread[nThreads];
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = new IndexingThread();
th.w = w;
th.base_Renamed = 1000000 * i;
th.range = range;
th.iterations = iterations;
threads[i] = th;
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Start();
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
// w.optimize();
//w.close();
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = threads[i];
lock (th)
{
SupportClass.CollectionsHelper.AddAllIfNotContains(docs, th.docs);
}
}
_TestUtil.CheckIndex(dir);
DocsAndWriter dw = new DocsAndWriter();
dw.docs = docs;
dw.writer = w;
return(dw);
}
public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir)
{
System.Collections.IDictionary docs = new System.Collections.Hashtable();
for (int iter = 0; iter < 3; iter++)
{
IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true);
w.SetUseCompoundFile(false);
// force many merges
w.SetMergeFactor(mergeFactor);
w.SetRAMBufferSizeMB(.1);
w.SetMaxBufferedDocs(maxBufferedDocs);
threads = new IndexingThread[nThreads];
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = new IndexingThread();
th.w = w;
th.base_Renamed = 1000000 * i;
th.range = range;
th.iterations = iterations;
threads[i] = th;
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Start();
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
// w.optimize();
w.Close();
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = threads[i];
lock (th)
{
System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator();
while (e.MoveNext())
{
docs[e.Current] = th.docs[e.Current];
}
}
}
}
_TestUtil.CheckIndex(dir);
return(docs);
}
public virtual void TestRandomExceptions()
{
MockRAMDirectory dir = new MockRAMDirectory();
MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions();
//writer.setMaxBufferedDocs(10);
writer.SetRAMBufferSizeMB(0.1);
if (DEBUG)
{
System.IO.StreamWriter temp_writer;
temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
temp_writer.AutoFlush = true;
writer.SetInfoStream(temp_writer);
}
IndexerThread thread = new IndexerThread(this, 0, writer);
thread.Run();
if (thread.failure != null)
{
System.Console.Out.WriteLine(thread.failure.StackTrace);
Assert.Fail("thread " + thread.Name + ": hit unexpected failure");
}
writer.Commit();
try
{
writer.Close();
}
catch (System.Exception t)
{
System.Console.Out.WriteLine("exception during close:");
System.Console.Out.WriteLine(t.StackTrace);
writer.Rollback();
}
// Confirm that when doc hits exception partway through tokenization, it's deleted:
IndexReader r2 = IndexReader.Open(dir);
int count = r2.DocFreq(new Term("content4", "aaa"));
int count2 = r2.DocFreq(new Term("content4", "ddd"));
Assert.AreEqual(count, count2);
r2.Close();
_TestUtil.CheckIndex(dir);
}
public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir)
{
System.Collections.IDictionary docs = new System.Collections.Hashtable();
for (int iter = 0; iter < 3; iter++)
{
IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true);
w.SetUseCompoundFile(false);
// force many merges
w.SetMergeFactor(mergeFactor);
w.SetRAMBufferSizeMB(.1);
w.SetMaxBufferedDocs(maxBufferedDocs);
threads = new IndexingThread[nThreads];
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = new IndexingThread();
th.w = w;
th.base_Renamed = 1000000 * i;
th.range = range;
th.iterations = iterations;
threads[i] = th;
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Start();
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
// w.optimize();
w.Close();
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = threads[i];
lock (th)
{
System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator();
while (e.MoveNext())
{
docs[e.Current] = th.docs[e.Current];
}
}
}
}
_TestUtil.CheckIndex(dir);
return docs;
}
public virtual DocsAndWriter IndexRandomIWReader(int nThreads, int iterations, int range, Directory dir)
{
System.Collections.Hashtable docs = new System.Collections.Hashtable();
IndexWriter w = new MockIndexWriter(this, dir, autoCommit, new WhitespaceAnalyzer(), true);
w.SetUseCompoundFile(false);
/***
w.setMaxMergeDocs(Integer.MAX_VALUE);
w.setMaxFieldLength(10000);
w.setRAMBufferSizeMB(1);
w.setMergeFactor(10);
***/
// force many merges
w.SetMergeFactor(mergeFactor);
w.SetRAMBufferSizeMB(.1);
w.SetMaxBufferedDocs(maxBufferedDocs);
threads = new IndexingThread[nThreads];
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = new IndexingThread();
th.w = w;
th.base_Renamed = 1000000 * i;
th.range = range;
th.iterations = iterations;
threads[i] = th;
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Start();
}
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
// w.optimize();
//w.close();
for (int i = 0; i < threads.Length; i++)
{
IndexingThread th = threads[i];
lock (th)
{
SupportClass.CollectionsHelper.AddAllIfNotContains(docs, th.docs);
}
}
_TestUtil.CheckIndex(dir);
DocsAndWriter dw = new DocsAndWriter();
dw.docs = docs;
dw.writer = w;
return dw;
}
public virtual void TestRandomExceptionsThreads()
{
MockRAMDirectory dir = new MockRAMDirectory();
MockIndexWriter writer = new MockIndexWriter(this, dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
((ConcurrentMergeScheduler) writer.GetMergeScheduler()).SetSuppressExceptions();
//writer.setMaxBufferedDocs(10);
writer.SetRAMBufferSizeMB(0.2);
if (DEBUG)
{
System.IO.StreamWriter temp_writer;
temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
temp_writer.AutoFlush = true;
writer.SetInfoStream(temp_writer);
}
int NUM_THREADS = 4;
IndexerThread[] threads = new IndexerThread[NUM_THREADS];
for (int i = 0; i < NUM_THREADS; i++)
{
threads[i] = new IndexerThread(this, i, writer);
threads[i].Start();
}
for (int i = 0; i < NUM_THREADS; i++)
threads[i].Join();
for (int i = 0; i < NUM_THREADS; i++)
if (threads[i].failure != null)
Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure");
writer.Commit();
try
{
writer.Close();
}
catch (System.Exception t)
{
System.Console.Out.WriteLine("exception during close:");
System.Console.Out.WriteLine(t.StackTrace);
writer.Rollback();
}
// Confirm that when doc hits exception partway through tokenization, it's deleted:
IndexReader r2 = IndexReader.Open(dir);
int count = r2.DocFreq(new Term("content4", "aaa"));
int count2 = r2.DocFreq(new Term("content4", "ddd"));
Assert.AreEqual(count, count2);
r2.Close();
_TestUtil.CheckIndex(dir);
}
