public override void Run()
{
try
{
r = new Random(@base + range + seed);
for (int i = 0; i < iterations; i++)
{
int what = NextInt(100);
if (what < 5)
{
DeleteDoc();
}
else if (what < 10)
{
DeleteByQuery();
}
else
{
IndexDoc();
}
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(e.ToString());
}
lock (this)
{
int dummy = docs.Count;
}
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
Console.WriteLine("NRT reopen times:");
for (int i = 0; i < reopenCount; i++)
{
Console.Write(" " + reopenTimes[i]);
}
Console.WriteLine();
}
}
public virtual void TestLaaaaaargeOutput()
{
string message = "I will not OOM on large output";
int howMuch = 250 * 1024 * 1024;
for (int i = 0; i < howMuch; i++)
{
if (i > 0)
{
Console.Write(",\n");
}
Console.Write(message);
howMuch -= message.Length; // approximately.
}
Console.WriteLine(".");
}
public override void Run()
{
try
{
for (int j = 0; j < numDocs; j++)
{
Document d = new Document();
d.Add(new TextField(field, new PoolingPayloadTokenStream(outerInstance, pool)));
writer.AddDocument(d);
}
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(e.ToString());
}
}
public override void Run()
{
try
{
signal.Wait();
manager.MaybeRefresh();
writer.DeleteDocuments(new TermQuery(new Term("foo", "barista")));
manager.MaybeRefresh(); // kick off another reopen so we inc. the internal gen
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
}
finally
{
latch.Reset(latch.CurrentCount == 0 ? 0 : latch.CurrentCount - 1); // let the add below finish
}
}
public override void Run()
{
try
{
r = new Random(@base + range + seed);
for (int i = 0; i < iterations; i++)
{
int what = NextInt(100);
if (what < 5)
{
DeleteDoc();
}
else if (what < 10)
{
DeleteByQuery();
}
else
{
IndexDoc();
}
}
}
catch (Exception e) when(e.IsThrowable())
{
Console.WriteLine(e.ToString());
Console.Write(e.StackTrace);
Assert.Fail(e.ToString());
}
UninterruptableMonitor.Enter(this);
try
{
int dummy = docs.Count;
}
finally
{
UninterruptableMonitor.Exit(this);
}
}
public override void Run()
{
// TODO: would be better if this were cross thread, so that we make sure one thread deleting anothers added docs works:
IList <string> toDeleteIDs = new List <string>();
IList <SubDocs> toDeleteSubDocs = new List <SubDocs>();
while (Environment.TickCount < stopTime && !outerInstance.m_failed)
{
try
{
// Occasional longish pause if running
// nightly
if (LuceneTestCase.TestNightly && Random.Next(6) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": now long sleep");
}
//Thread.Sleep(TestUtil.NextInt32(Random, 50, 500));
// LUCENENET specific - Reduced amount of pause to keep the total
// Nightly test time under 1 hour
Thread.Sleep(TestUtil.NextInt32(Random, 50, 250));
}
// Rate limit ingest rate:
if (Random.Next(7) == 5)
{
Thread.Sleep(TestUtil.NextInt32(Random, 1, 10));
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": done sleep");
}
}
Document doc = docs.NextDoc();
if (doc == null)
{
break;
}
// Maybe add randomly named field
string addedField;
if (Random.NextBoolean())
{
addedField = "extra" + Random.Next(40);
doc.Add(NewTextField(addedField, "a random field", Field.Store.YES));
}
else
{
addedField = null;
}
if (Random.NextBoolean())
{
if (Random.NextBoolean())
{
// Add/update doc block:
string packID;
SubDocs delSubDocs;
if (toDeleteSubDocs.Count > 0 && Random.NextBoolean())
{
delSubDocs = toDeleteSubDocs[Random.Next(toDeleteSubDocs.Count)];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!delSubDocs.Deleted);
}
toDeleteSubDocs.Remove(delSubDocs);
// Update doc block, replacing prior packID
packID = delSubDocs.PackID;
}
else
{
delSubDocs = null;
// Add doc block, using new packID
packID = outerInstance.m_packCount.GetAndIncrement().ToString(CultureInfo.InvariantCulture);
}
Field packIDField = NewStringField("packID", packID, Field.Store.YES);
IList <string> docIDs = new List <string>();
SubDocs subDocs = new SubDocs(packID, docIDs);
IList <Document> docsList = new List <Document>();
allSubDocs.Enqueue(subDocs);
doc.Add(packIDField);
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
int maxDocCount = TestUtil.NextInt32(Random, 1, 10);
while (docsList.Count < maxDocCount)
{
doc = docs.NextDoc();
if (doc == null)
{
break;
}
docsList.Add(TestUtil.CloneDocument(doc));
docIDs.Add(doc.Get("docid"));
}
outerInstance.m_addCount.AddAndGet(docsList.Count);
Term packIDTerm = new Term("packID", packID);
if (delSubDocs != null)
{
delSubDocs.Deleted = true;
delIDs.UnionWith(delSubDocs.SubIDs);
outerInstance.m_delCount.AddAndGet(delSubDocs.SubIDs.Count);
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update pack packID=" + delSubDocs.PackID +
" count=" + docsList.Count + " docs=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", docIDs));
}
outerInstance.UpdateDocuments(packIDTerm, docsList);
}
else
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add pack packID=" + packID +
" count=" + docsList.Count + " docs=" + string.Format(J2N.Text.StringFormatter.InvariantCulture, "{0}", docIDs));
}
outerInstance.AddDocuments(packIDTerm, docsList);
}
doc.RemoveField("packID");
if (Random.Next(5) == 2)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + packID);
}
toDeleteSubDocs.Add(subDocs);
}
}
else
{
// Add single doc
string docid = doc.Get("docid");
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": add doc docid:" + docid);
}
outerInstance.AddDocument(new Term("docid", docid), doc);
outerInstance.m_addCount.GetAndIncrement();
if (Random.Next(5) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
}
else
{
// Update single doc, but we never re-use
// and ID so the delete will never
// actually happen:
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": update doc id:" + doc.Get("docid"));
}
string docid = doc.Get("docid");
outerInstance.UpdateDocument(new Term("docid", docid), doc);
outerInstance.m_addCount.GetAndIncrement();
if (Random.Next(5) == 3)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": buffer del id:" + doc.Get("docid"));
}
toDeleteIDs.Add(docid);
}
}
if (Random.Next(30) == 17)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": apply " + toDeleteIDs.Count + " deletes");
}
foreach (string id in toDeleteIDs)
{
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del term=id:" + id);
}
outerInstance.DeleteDocuments(new Term("docid", id));
}
int count = outerInstance.m_delCount.AddAndGet(toDeleteIDs.Count);
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": tot " + count + " deletes");
}
delIDs.UnionWith(toDeleteIDs);
toDeleteIDs.Clear();
foreach (SubDocs subDocs in toDeleteSubDocs)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(!subDocs.Deleted);
}
delPackIDs.Add(subDocs.PackID);
outerInstance.DeleteDocuments(new Term("packID", subDocs.PackID));
subDocs.Deleted = true;
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": del subs: " + subDocs.SubIDs + " packID=" + subDocs.PackID);
}
delIDs.UnionWith(subDocs.SubIDs);
outerInstance.m_delCount.AddAndGet(subDocs.SubIDs.Count);
}
toDeleteSubDocs.Clear();
}
if (addedField != null)
{
doc.RemoveField(addedField);
}
}
catch (Exception t)
{
Console.WriteLine(Thread.CurrentThread.Name + ": hit exc");
Console.WriteLine(t.ToString());
Console.Write(t.StackTrace);
outerInstance.m_failed.Value = (true);
throw new Exception(t.ToString(), t);
}
}
if (Verbose)
{
Console.WriteLine(Thread.CurrentThread.Name + ": indexing done");
}
outerInstance.DoAfterIndexingThreadDone();
}
public void TestRandom()
{
string[] terms = new string[TestUtil.NextInt32(Random, 2, 10)];
ISet <string> seen = new JCG.HashSet <string>();
while (seen.size() < terms.Length)
{
string token = TestUtil.RandomSimpleString(Random, 1, 5);
if (!seen.contains(token))
{
terms[seen.size()] = token;
seen.add(token);
}
}
Analyzer a = new MockAnalyzer(Random);
int numDocs = AtLeast(10);
long totTokens = 0;
string[][] docs = new string[numDocs][];
for (int i = 0; i < numDocs; i++)
{
docs[i] = new string[AtLeast(100)];
if (Verbose)
{
Console.Write(" doc " + i + ":");
}
for (int j = 0; j < docs[i].Length; j++)
{
docs[i][j] = GetZipfToken(terms);
if (Verbose)
{
Console.Write(" " + docs[i][j]);
}
}
if (Verbose)
{
Console.WriteLine();
}
totTokens += docs[i].Length;
}
int grams = TestUtil.NextInt32(Random, 1, 4);
if (Verbose)
{
Console.WriteLine("TEST: " + terms.Length + " terms; " + numDocs + " docs; " + grams + " grams");
}
// Build suggester model:
FreeTextSuggester sug = new FreeTextSuggester(a, a, grams, (byte)0x20);
sug.Build(new TestRandomInputIterator(this, docs));
// Build inefficient but hopefully correct model:
List <IDictionary <string, int?> > gramCounts = new List <IDictionary <string, int?> >(grams);
for (int gram = 0; gram < grams; gram++)
{
if (Verbose)
{
Console.WriteLine("TEST: build model for gram=" + gram);
}
IDictionary <string, int?> model = new JCG.Dictionary <string, int?>();
gramCounts.Add(model);
foreach (string[] doc in docs)
{
for (int i = 0; i < doc.Length - gram; i++)
{
StringBuilder b = new StringBuilder();
for (int j = i; j <= i + gram; j++)
{
if (j > i)
{
b.append(' ');
}
b.append(doc[j]);
}
string token = b.toString();
if (!model.TryGetValue(token, out int?curCount) || curCount == null)
{
model.Put(token, 1);
}
else
{
model.Put(token, 1 + curCount);
}
if (Verbose)
{
Console.WriteLine(" add '" + token + "' -> count=" + (model.TryGetValue(token, out int?count) ? (count.HasValue ? count.ToString() : "null") : ""));
}
}
}
}
int lookups = AtLeast(100);
for (int iter = 0; iter < lookups; iter++)
{
string[] tokens = new string[TestUtil.NextInt32(Random, 1, 5)];
for (int i = 0; i < tokens.Length; i++)
{
tokens[i] = GetZipfToken(terms);
}
// Maybe trim last token; be sure not to create the
// empty string:
int trimStart;
if (tokens.Length == 1)
{
trimStart = 1;
}
else
{
trimStart = 0;
}
int trimAt = TestUtil.NextInt32(Random, trimStart, tokens[tokens.Length - 1].Length);
tokens[tokens.Length - 1] = tokens[tokens.Length - 1].Substring(0, trimAt - 0);
int num = TestUtil.NextInt32(Random, 1, 100);
StringBuilder b = new StringBuilder();
foreach (string token in tokens)
{
b.append(' ');
b.append(token);
}
string query = b.toString();
query = query.Substring(1);
if (Verbose)
{
Console.WriteLine("\nTEST: iter=" + iter + " query='" + query + "' num=" + num);
}
// Expected:
List <Lookup.LookupResult> expected = new List <Lookup.LookupResult>();
double backoff = 1.0;
seen = new JCG.HashSet <string>();
if (Verbose)
{
Console.WriteLine(" compute expected");
}
for (int i = grams - 1; i >= 0; i--)
{
if (Verbose)
{
Console.WriteLine(" grams=" + i);
}
if (tokens.Length < i + 1)
{
// Don't have enough tokens to use this model
if (Verbose)
{
Console.WriteLine(" skip");
}
continue;
}
if (i == 0 && tokens[tokens.Length - 1].Length == 0)
{
// Never suggest unigrams from empty string:
if (Verbose)
{
Console.WriteLine(" skip unigram priors only");
}
continue;
}
// Build up "context" ngram:
b = new StringBuilder();
for (int j = tokens.Length - i - 1; j < tokens.Length - 1; j++)
{
b.append(' ');
b.append(tokens[j]);
}
string context = b.toString();
if (context.Length > 0)
{
context = context.Substring(1);
}
if (Verbose)
{
Console.WriteLine(" context='" + context + "'");
}
long contextCount;
if (context.Length == 0)
{
contextCount = totTokens;
}
else
{
//int? count = gramCounts.get(i - 1).get(context);
var gramCount = gramCounts[i - 1];
if (!gramCount.TryGetValue(context, out int?count) || count == null)
{
// We never saw this context:
backoff *= FreeTextSuggester.ALPHA;
if (Verbose)
{
Console.WriteLine(" skip: never saw context");
}
continue;
}
contextCount = count.GetValueOrDefault();
}
if (Verbose)
{
Console.WriteLine(" contextCount=" + contextCount);
}
IDictionary <string, int?> model = gramCounts[i];
// First pass, gather all predictions for this model:
if (Verbose)
{
Console.WriteLine(" find terms w/ prefix=" + tokens[tokens.Length - 1]);
}
List <Lookup.LookupResult> tmp = new List <Lookup.LookupResult>();
foreach (string term in terms)
{
if (term.StartsWith(tokens[tokens.Length - 1], StringComparison.Ordinal))
{
if (Verbose)
{
Console.WriteLine(" term=" + term);
}
if (seen.contains(term))
{
if (Verbose)
{
Console.WriteLine(" skip seen");
}
continue;
}
string ngram = (context + " " + term).Trim();
//Integer count = model.get(ngram);
if (model.TryGetValue(ngram, out int?count) && count != null)
{
// LUCENENET NOTE: We need to calculate this as decimal because when using double it can sometimes
// return numbers that are greater than long.MaxValue, which results in a negative long number.
// This is also the way it is being done in the FreeTextSuggester to work around the issue.
Lookup.LookupResult lr = new Lookup.LookupResult(ngram, (long)(long.MaxValue * ((decimal)backoff * (decimal)count / contextCount)));
tmp.Add(lr);
if (Verbose)
{
Console.WriteLine(" add tmp key='" + lr.Key + "' score=" + lr.Value);
}
}
}
}
// Second pass, trim to only top N, and fold those
// into overall suggestions:
tmp.Sort(byScoreThenKey);
if (tmp.size() > num)
{
//tmp.subList(num, tmp.size()).clear();
tmp.RemoveRange(num, tmp.size() - num);
}
foreach (Lookup.LookupResult result in tmp)
{
string key = result.Key.toString();
int idx = key.LastIndexOf(' ');
string lastToken;
if (idx != -1)
{
lastToken = key.Substring(idx + 1);
}
else
{
lastToken = key;
}
if (!seen.contains(lastToken))
{
seen.add(lastToken);
expected.Add(result);
if (Verbose)
{
Console.WriteLine(" keep key='" + result.Key + "' score=" + result.Value);
}
}
}
backoff *= FreeTextSuggester.ALPHA;
}
expected.Sort(byScoreThenKey);
if (expected.size() > num)
{
expected.RemoveRange(num, expected.size() - num);
}
// Actual:
IList <Lookup.LookupResult> actual = sug.DoLookup(query, num);
if (Verbose)
{
Console.WriteLine(" expected: " + expected);
Console.WriteLine(" actual: " + actual);
}
assertEquals(expected.ToString(), actual.ToString());
}
}
public virtual void TestRandomMaps2()
{
Random random = Random;
int numIterations = AtLeast(3);
for (int iter = 0; iter < numIterations; iter++)
{
if (Verbose)
{
Console.WriteLine("\nTEST iter=" + iter);
}
char endLetter = (char)TestUtil.NextInt32(random, 'b', 'z');
IDictionary <string, string> map = new Dictionary <string, string>();
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
int numMappings = AtLeast(5);
if (Verbose)
{
Console.WriteLine(" mappings:");
}
while (map.Count < numMappings)
{
string key = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, 7);
if (key.Length != 0 && !map.ContainsKey(key))
{
string value = TestUtil.RandomSimpleString(random);
map[key] = value;
builder.Add(key, value);
if (Verbose)
{
Console.WriteLine(" " + key + " -> " + value);
}
}
}
NormalizeCharMap charMap = builder.Build();
if (Verbose)
{
Console.WriteLine(" test random documents...");
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
string content = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, AtLeast(1000));
if (Verbose)
{
Console.WriteLine(" content=" + content);
}
// Do stupid dog-slow mapping:
// Output string:
StringBuilder output = new StringBuilder();
// Maps output offset to input offset:
IList <int?> inputOffsets = new List <int?>();
int cumDiff = 0;
int charIdx = 0;
while (charIdx < content.Length)
{
int matchLen = -1;
string matchRepl = null;
foreach (KeyValuePair <string, string> ent in map)
{
string match = ent.Key;
if (charIdx + match.Length <= content.Length)
{
int limit = charIdx + match.Length;
bool matches = true;
for (int charIdx2 = charIdx; charIdx2 < limit; charIdx2++)
{
if (match[charIdx2 - charIdx] != content[charIdx2])
{
matches = false;
break;
}
}
if (matches)
{
string repl = ent.Value;
if (match.Length > matchLen)
{
// Greedy: longer match wins
matchLen = match.Length;
matchRepl = repl;
}
}
}
}
if (matchLen != -1)
{
// We found a match here!
if (Verbose)
{
Console.WriteLine(" match=" + content.Substring(charIdx, matchLen) + " @ off=" + charIdx + " repl=" + matchRepl);
}
output.Append(matchRepl);
int minLen = Math.Min(matchLen, matchRepl.Length);
// Common part, directly maps back to input
// offset:
for (int outIdx = 0; outIdx < minLen; outIdx++)
{
inputOffsets.Add(output.Length - matchRepl.Length + outIdx + cumDiff);
}
cumDiff += matchLen - matchRepl.Length;
charIdx += matchLen;
if (matchRepl.Length < matchLen)
{
// Replacement string is shorter than matched
// input: nothing to do
}
else if (matchRepl.Length > matchLen)
{
// Replacement string is longer than matched
// input: for all the "extra" chars we map
// back to a single input offset:
for (int outIdx = matchLen; outIdx < matchRepl.Length; outIdx++)
{
inputOffsets.Add(output.Length + cumDiff - 1);
}
}
else
{
// Same length: no change to offset
}
if (Debugging.AssertsEnabled)
{
Debugging.Assert(inputOffsets.Count == output.Length, () => "inputOffsets.size()=" + inputOffsets.Count + " vs output.length()=" + output.Length);
}
}
else
{
inputOffsets.Add(output.Length + cumDiff);
output.Append(content[charIdx]);
charIdx++;
}
}
string expected = output.ToString();
if (Verbose)
{
Console.Write(" expected:");
for (int charIdx2 = 0; charIdx2 < expected.Length; charIdx2++)
{
Console.Write(" " + expected[charIdx2] + "/" + inputOffsets[charIdx2]);
}
Console.WriteLine();
}
MappingCharFilter mapFilter = new MappingCharFilter(charMap, new StringReader(content));
StringBuilder actualBuilder = new StringBuilder();
IList <int?> actualInputOffsets = new List <int?>();
// Now consume the actual mapFilter, somewhat randomly:
while (true)
{
if (random.Next(0, 1) == 1)
{
int ch = mapFilter.Read();
if (ch == -1)
{
break;
}
actualBuilder.Append((char)ch);
}
else
{
char[] buffer = new char[TestUtil.NextInt32(random, 1, 100)];
int off = buffer.Length == 1 ? 0 : random.Next(buffer.Length - 1);
int count = mapFilter.Read(buffer, off, buffer.Length - off);
if (count == -1)
{
break;
}
else
{
actualBuilder.Append(buffer, off, count);
}
}
if (random.Next(10) == 7)
{
// Map offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
}
}
// Finish mappping offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
string actual = actualBuilder.ToString();
// Verify:
assertEquals(expected, actual);
assertEquals(inputOffsets, actualInputOffsets, aggressive: false);
}
}
}
