public void Sort()
{
SubDocs.Sort(new DocComparer());
foreach (var doc in SubDocs)
{
doc.Sort();
}
}
public override void Run()
{
// TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
IList <string> toDeleteIDs = new List <string>();
IList <SubDocs> toDeleteSubDocs = new List <SubDocs>();
while (DateTime.UtcNow < StopTime && !OuterInstance.Failed.Get())
{
try
{
// Occasional longish pause if running
// nightly
if (LuceneTestCase.TEST_NIGHTLY && Random().Next(6) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": now long sleep");
}
Thread.Sleep(TestUtil.NextInt(Random(), 50, 500));
}
// Rate limit ingest rate:
if (Random().Next(7) == 5)
{
Thread.Sleep(TestUtil.NextInt(Random(), 1, 10));
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": done sleep");
}
}
Document doc = Docs.NextDoc();
if (doc == null)
{
break;
}
// Maybe add randomly named field
string addedField;
if (Random().NextBoolean())
{
addedField = "extra" + Random().Next(40);
doc.Add(NewTextField(addedField, "a random field", Field.Store.YES));
}
else
{
addedField = null;
}
if (Random().NextBoolean())
{
if (Random().NextBoolean())
{
// Add/update doc block:
string packID;
SubDocs delSubDocs;
if (toDeleteSubDocs.Count > 0 && Random().NextBoolean())
{
delSubDocs = toDeleteSubDocs[Random().Next(toDeleteSubDocs.Count)];
Debug.Assert(!delSubDocs.Deleted);
toDeleteSubDocs.Remove(delSubDocs);
// Update doc block, replacing prior packID
packID = delSubDocs.PackID;
}
else
{
delSubDocs = null;
// Add doc block, using new packID
packID = OuterInstance.PackCount.IncrementAndGet() + "";
}
Field packIDField = NewStringField("packID", packID, Field.Store.YES);
IList <string> docIDs = new List <string>();
SubDocs subDocs = new SubDocs(packID, docIDs);
IList <Document> docsList = new List <Document>();
AllSubDocs.Add(subDocs);
doc.Add(packIDField);
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
int maxDocCount = TestUtil.NextInt(Random(), 1, 10);
while (docsList.Count < maxDocCount)
{
doc = Docs.NextDoc();
if (doc == null)
{
break;
}
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
}
OuterInstance.AddCount.AddAndGet(docsList.Count);
Term packIDTerm = new Term("packID", packID);
if (delSubDocs != null)
{
delSubDocs.Deleted = true;
DelIDs.AddAll(delSubDocs.SubIDs);
OuterInstance.DelCount.AddAndGet(delSubDocs.SubIDs.Count);
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update pack packID=" + delSubDocs.PackID + " count=" + docsList.Count + " docs=" + docIDs);
}
OuterInstance.UpdateDocuments(packIDTerm, docsList);
}
else
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add pack packID=" + packID + " count=" + docsList.Count + " docs=" + docIDs);
}
OuterInstance.AddDocuments(packIDTerm, docsList);
}
doc.RemoveField("packID");
if (Random().Next(5) == 2)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + packID);
}
toDeleteSubDocs.Add(subDocs);
}
}
else
{
// Add single doc
string docid = doc.Get("docid");
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add doc docid:" + docid);
}
OuterInstance.AddDocument(new Term("docid", docid), doc);
OuterInstance.AddCount.IncrementAndGet();
if (Random().Next(5) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
}
else
{
// Update single doc, but we never re-use
// and ID so the delete will never
// actually happen:
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update doc id:" + doc.Get("docid"));
}
string docid = doc.Get("docid");
OuterInstance.UpdateDocument(new Term("docid", docid), doc);
OuterInstance.AddCount.IncrementAndGet();
if (Random().Next(5) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
if (Random().Next(30) == 17)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": apply " + toDeleteIDs.Count + " deletes");
}
foreach (string id in toDeleteIDs)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del term=id:" + id);
}
OuterInstance.DeleteDocuments(new Term("docid", id));
}
int count = OuterInstance.DelCount.AddAndGet(toDeleteIDs.Count);
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": tot " + count + " deletes");
}
DelIDs.AddAll(toDeleteIDs);
toDeleteIDs.Clear();
foreach (SubDocs subDocs in toDeleteSubDocs)
{
Debug.Assert(!subDocs.Deleted);
DelPackIDs.Add(subDocs.PackID);
OuterInstance.DeleteDocuments(new Term("packID", subDocs.PackID));
subDocs.Deleted = true;
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del subs: " + subDocs.SubIDs + " packID=" + subDocs.PackID);
}
DelIDs.AddAll(subDocs.SubIDs);
OuterInstance.DelCount.AddAndGet(subDocs.SubIDs.Count);
}
toDeleteSubDocs.Clear();
}
if (addedField != null)
{
doc.RemoveField(addedField);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
OuterInstance.Failed.Set(true);
throw new Exception(t.Message, t);
}
}
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": indexing done");
}
OuterInstance.DoAfterIndexingThreadDone();
}
public override void Run()
{
// TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
IList <string> toDeleteIDs = new List <string>();
IList <SubDocs> toDeleteSubDocs = new List <SubDocs>();
while (Environment.TickCount < stopTime && !outerInstance.m_failed)
{
try
{
// Occasional longish pause if running
// nightly
if (LuceneTestCase.TestNightly && Random.Next(6) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": now long sleep");
}
//Thread.Sleep(TestUtil.NextInt32(Random, 50, 500));
// LUCENENET specific - Reduced amount of pause to keep the total
// Nightly test time under 1 hour
Thread.Sleep(TestUtil.NextInt32(Random, 50, 250));
}
// Rate limit ingest rate:
if (Random.Next(7) == 5)
{
Thread.Sleep(TestUtil.NextInt32(Random, 1, 10));
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": done sleep");
}
}
Document doc = docs.NextDoc();
if (doc == null)
{
break;
}
// Maybe add randomly named field
string addedField;
if (Random.NextBoolean())
{
addedField = "extra" + Random.Next(40);
doc.Add(NewTextField(addedField, "a random field", Field.Store.YES));
}
else
{
addedField = null;
}
if (Random.NextBoolean())
{
if (Random.NextBoolean())
{
// Add/update doc block:
string packID;
SubDocs delSubDocs;
if (toDeleteSubDocs.Count > 0 && Random.NextBoolean())
{
delSubDocs = toDeleteSubDocs[Random.Next(toDeleteSubDocs.Count)];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!delSubDocs.Deleted);
}
toDeleteSubDocs.Remove(delSubDocs);
// Update doc block, replacing prior packID
packID = delSubDocs.PackID;
}
else
{
delSubDocs = null;
// Add doc block, using new packID
packID = outerInstance.m_packCount.GetAndIncrement().ToString(CultureInfo.InvariantCulture);
}
Field packIDField = NewStringField("packID", packID, Field.Store.YES);
IList <string> docIDs = new List <string>();
SubDocs subDocs = new SubDocs(packID, docIDs);
IList <Document> docsList = new List <Document>();
allSubDocs.Enqueue(subDocs);
doc.Add(packIDField);
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
int maxDocCount = TestUtil.NextInt32(Random, 1, 10);
while (docsList.Count < maxDocCount)
{
doc = docs.NextDoc();
if (doc == null)
{
break;
}
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
}
outerInstance.m_addCount.AddAndGet(docsList.Count);
Term packIDTerm = new Term("packID", packID);
if (delSubDocs != null)
{
delSubDocs.Deleted = true;
delIDs.UnionWith(delSubDocs.SubIDs);
outerInstance.m_delCount.AddAndGet(delSubDocs.SubIDs.Count);
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update pack packID=" + delSubDocs.PackID +
" count=" + docsList.Count + " docs=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", docIDs));
}
outerInstance.UpdateDocuments(packIDTerm, docsList);
}
else
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add pack packID=" + packID +
" count=" + docsList.Count + " docs=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", docIDs));
}
outerInstance.AddDocuments(packIDTerm, docsList);
}
doc.RemoveField("packID");
if (Random.Next(5) == 2)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + packID);
}
toDeleteSubDocs.Add(subDocs);
}
}
else
{
// Add single doc
string docid = doc.Get("docid");
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add doc docid:" + docid);
}
outerInstance.AddDocument(new Term("docid", docid), doc);
outerInstance.m_addCount.GetAndIncrement();
if (Random.Next(5) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
}
else
{
// Update single doc, but we never re-use
// and ID so the delete will never
// actually happen:
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update doc id:" + doc.Get("docid"));
}
string docid = doc.Get("docid");
outerInstance.UpdateDocument(new Term("docid", docid), doc);
outerInstance.m_addCount.GetAndIncrement();
if (Random.Next(5) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
if (Random.Next(30) == 17)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": apply " + toDeleteIDs.Count + " deletes");
}
foreach (string id in toDeleteIDs)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del term=id:" + id);
}
outerInstance.DeleteDocuments(new Term("docid", id));
}
int count = outerInstance.m_delCount.AddAndGet(toDeleteIDs.Count);
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": tot " + count + " deletes");
}
delIDs.UnionWith(toDeleteIDs);
toDeleteIDs.Clear();
foreach (SubDocs subDocs in toDeleteSubDocs)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!subDocs.Deleted);
}
delPackIDs.Add(subDocs.PackID);
outerInstance.DeleteDocuments(new Term("packID", subDocs.PackID));
subDocs.Deleted = true;
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del subs: " + subDocs.SubIDs + " packID=" + subDocs.PackID);
}
delIDs.UnionWith(subDocs.SubIDs);
outerInstance.m_delCount.AddAndGet(subDocs.SubIDs.Count);
}
toDeleteSubDocs.Clear();
}
if (addedField != null)
{
doc.RemoveField(addedField);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
outerInstance.m_failed.Value = (true);
throw new Exception(t.ToString(), t);
}
}
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": indexing done");
}
outerInstance.DoAfterIndexingThreadDone();
}
public override void Run()
{
// TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
IList<string> toDeleteIDs = new List<string>();
IList<SubDocs> toDeleteSubDocs = new List<SubDocs>();
while (DateTime.UtcNow < StopTime && !OuterInstance.Failed.Get())
{
try
{
// Occasional longish pause if running
// nightly
if (LuceneTestCase.TEST_NIGHTLY && Random().Next(6) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": now long sleep");
}
Thread.Sleep(TestUtil.NextInt(Random(), 50, 500));
}
// Rate limit ingest rate:
if (Random().Next(7) == 5)
{
Thread.Sleep(TestUtil.NextInt(Random(), 1, 10));
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": done sleep");
}
}
Document doc = Docs.NextDoc();
if (doc == null)
{
break;
}
// Maybe add randomly named field
string addedField;
if (Random().NextBoolean())
{
addedField = "extra" + Random().Next(40);
doc.Add(OuterInstance.NewTextField(addedField, "a random field", Field.Store.YES));
}
else
{
addedField = null;
}
if (Random().NextBoolean())
{
if (Random().NextBoolean())
{
// Add/update doc block:
string packID;
SubDocs delSubDocs;
if (toDeleteSubDocs.Count > 0 && Random().NextBoolean())
{
delSubDocs = toDeleteSubDocs[Random().Next(toDeleteSubDocs.Count)];
Debug.Assert(!delSubDocs.Deleted);
toDeleteSubDocs.Remove(delSubDocs);
// Update doc block, replacing prior packID
packID = delSubDocs.PackID;
}
else
{
delSubDocs = null;
// Add doc block, using new packID
packID = OuterInstance.PackCount.IncrementAndGet() + "";
}
Field packIDField = OuterInstance.NewStringField("packID", packID, Field.Store.YES);
IList<string> docIDs = new List<string>();
SubDocs subDocs = new SubDocs(packID, docIDs);
IList<Document> docsList = new List<Document>();
AllSubDocs.Add(subDocs);
doc.Add(packIDField);
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
int maxDocCount = TestUtil.NextInt(Random(), 1, 10);
while (docsList.Count < maxDocCount)
{
doc = Docs.NextDoc();
if (doc == null)
{
break;
}
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
}
OuterInstance.AddCount.AddAndGet(docsList.Count);
Term packIDTerm = new Term("packID", packID);
if (delSubDocs != null)
{
delSubDocs.Deleted = true;
DelIDs.AddAll(delSubDocs.SubIDs);
OuterInstance.DelCount.AddAndGet(delSubDocs.SubIDs.Count);
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update pack packID=" + delSubDocs.PackID + " count=" + docsList.Count + " docs=" + docIDs);
}
OuterInstance.UpdateDocuments(packIDTerm, docsList);
}
else
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add pack packID=" + packID + " count=" + docsList.Count + " docs=" + docIDs);
}
OuterInstance.AddDocuments(packIDTerm, docsList);
}
doc.RemoveField("packID");
if (Random().Next(5) == 2)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + packID);
}
toDeleteSubDocs.Add(subDocs);
}
}
else
{
// Add single doc
string docid = doc.Get("docid");
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add doc docid:" + docid);
}
OuterInstance.AddDocument(new Term("docid", docid), doc);
OuterInstance.AddCount.IncrementAndGet();
if (Random().Next(5) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
}
else
{
// Update single doc, but we never re-use
// and ID so the delete will never
// actually happen:
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update doc id:" + doc.Get("docid"));
}
string docid = doc.Get("docid");
OuterInstance.UpdateDocument(new Term("docid", docid), doc);
OuterInstance.AddCount.IncrementAndGet();
if (Random().Next(5) == 3)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
if (Random().Next(30) == 17)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": apply " + toDeleteIDs.Count + " deletes");
}
foreach (string id in toDeleteIDs)
{
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del term=id:" + id);
}
OuterInstance.DeleteDocuments(new Term("docid", id));
}
int count = OuterInstance.DelCount.AddAndGet(toDeleteIDs.Count);
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": tot " + count + " deletes");
}
DelIDs.AddAll(toDeleteIDs);
toDeleteIDs.Clear();
foreach (SubDocs subDocs in toDeleteSubDocs)
{
Debug.Assert(!subDocs.Deleted);
DelPackIDs.Add(subDocs.PackID);
OuterInstance.DeleteDocuments(new Term("packID", subDocs.PackID));
subDocs.Deleted = true;
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del subs: " + subDocs.SubIDs + " packID=" + subDocs.PackID);
}
DelIDs.AddAll(subDocs.SubIDs);
OuterInstance.DelCount.AddAndGet(subDocs.SubIDs.Count);
}
toDeleteSubDocs.Clear();
}
if (addedField != null)
{
doc.RemoveField(addedField);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
OuterInstance.Failed.Set(true);
throw new Exception(t.Message, t);
}
}
if (VERBOSE)
{
Console.WriteLine(Thread.CurrentThread.Name + ": indexing done");
}
OuterInstance.DoAfterIndexingThreadDone();
}