public virtual void TestBS2DisjunctionNextVsAdvance()
{
Directory d = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), d, Similarity, TimeZone);
int numDocs = AtLeast(300);
for (int docUpto = 0; docUpto < numDocs; docUpto++)
{
string contents = "a";
if (Random().Next(20) <= 16)
{
contents += " b";
}
if (Random().Next(20) <= 8)
{
contents += " c";
}
if (Random().Next(20) <= 4)
{
contents += " d";
}
if (Random().Next(20) <= 2)
{
contents += " e";
}
if (Random().Next(20) <= 1)
{
contents += " f";
}
Document doc = new Document();
doc.Add(new TextField("field", contents, Field.Store.NO));
w.AddDocument(doc);
}
w.ForceMerge(1);
IndexReader r = w.Reader;
IndexSearcher s = NewSearcher(r);
w.Dispose();
for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
{
if (VERBOSE)
{
Console.WriteLine("iter=" + iter);
}
IList <string> terms = new List <string>(Arrays.AsList("a", "b", "c", "d", "e", "f"));
int numTerms = TestUtil.NextInt(Random(), 1, terms.Count);
while (terms.Count > numTerms)
{
terms.RemoveAt(Random().Next(terms.Count));
}
if (VERBOSE)
{
Console.WriteLine(" terms=" + terms);
}
BooleanQuery q = new BooleanQuery();
foreach (string term in terms)
{
q.Add(new BooleanClause(new TermQuery(new Term("field", term)), Occur.SHOULD));
}
Weight weight = s.CreateNormalizedWeight(q);
Scorer scorer = weight.GetScorer(s.m_leafContexts[0], null);
// First pass: just use .NextDoc() to gather all hits
IList <ScoreDoc> hits = new List <ScoreDoc>();
while (scorer.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
{
hits.Add(new ScoreDoc(scorer.DocID, scorer.GetScore()));
}
if (VERBOSE)
{
Console.WriteLine(" " + hits.Count + " hits");
}
// Now, randomly next/advance through the list and
// verify exact match:
for (int iter2 = 0; iter2 < 10; iter2++)
{
weight = s.CreateNormalizedWeight(q);
scorer = weight.GetScorer(s.m_leafContexts[0], null);
if (VERBOSE)
{
Console.WriteLine(" iter2=" + iter2);
}
int upto = -1;
while (upto < hits.Count)
{
int nextUpto;
int nextDoc;
int left = hits.Count - upto;
if (left == 1 || Random().nextBoolean())
{
// next
nextUpto = 1 + upto;
nextDoc = scorer.NextDoc();
}
else
{
// advance
int inc = TestUtil.NextInt(Random(), 1, left - 1);
nextUpto = inc + upto;
nextDoc = scorer.Advance(hits[nextUpto].Doc);
}
if (nextUpto == hits.Count)
{
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, nextDoc);
}
else
{
ScoreDoc hit = hits[nextUpto];
Assert.AreEqual(hit.Doc, nextDoc);
// Test for precise float equality:
Assert.IsTrue(hit.Score == scorer.GetScore(), "doc " + hit.Doc + " has wrong score: expected=" + hit.Score + " actual=" + scorer.GetScore());
}
upto = nextUpto;
}
}
}
r.Dispose();
d.Dispose();
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadJob[numReleasers + numStallers + numWaiters];
IList <Exception> exceptions = new SynchronizedList <Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint)
{
Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked && ctrl.IsHealthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Value = (false);
sync.Waiter.Signal();
sync.LeftCheckpoint.Wait();
}
Assert.IsFalse(checkPoint);
Assert.AreEqual(0, sync.Waiter.CurrentCount);
if (checkPointProbability >= (float)Random.NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
}
if (!checkPoint)
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Value = (false);
stop.Value = (true);
sync.Waiter.Signal();
sync.LeftCheckpoint.Wait();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public virtual void TestRandom()
{
string[] tokens = GetRandomTokens(10);
Store.Directory indexDir = NewDirectory();
Store.Directory taxoDir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, indexDir);
var tw = new DirectoryTaxonomyWriter(taxoDir);
FacetsConfig config = new FacetsConfig();
int numDocs = AtLeast(1000);
int numDims = TestUtil.NextInt32(Random, 1, 7);
IList <TestDoc> testDocs = GetRandomDocs(tokens, numDocs, numDims);
foreach (TestDoc testDoc in testDocs)
{
Document doc = new Document();
doc.Add(NewStringField("content", testDoc.content, Field.Store.NO));
testDoc.value = Random.NextSingle();
doc.Add(new SingleDocValuesField("value", testDoc.value));
for (int j = 0; j < numDims; j++)
{
if (testDoc.dims[j] != null)
{
doc.Add(new FacetField("dim" + j, testDoc.dims[j]));
}
}
w.AddDocument(config.Build(tw, doc));
}
// NRT open
IndexSearcher searcher = NewSearcher(w.GetReader());
// NRT open
var tr = new DirectoryTaxonomyReader(tw);
ValueSource values = new SingleFieldSource("value");
int iters = AtLeast(100);
for (int iter = 0; iter < iters; iter++)
{
string searchToken = tokens[Random.Next(tokens.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter content=" + searchToken);
}
FacetsCollector fc = new FacetsCollector();
FacetsCollector.Search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
Facets facets = new TaxonomyFacetSumValueSource(tr, config, fc, values);
// Slow, yet hopefully bug-free, faceting:
var expectedValues = new List <Dictionary <string, float?> >(numDims);
for (int i = 0; i < numDims; i++)
{
expectedValues.Add(new Dictionary <string, float?>());
}
foreach (TestDoc doc in testDocs)
{
if (doc.content.Equals(searchToken, StringComparison.Ordinal))
{
for (int j = 0; j < numDims; j++)
{
if (doc.dims[j] != null)
{
if (!expectedValues[j].TryGetValue(doc.dims[j], out float?v) || v == null)
{
expectedValues[j][doc.dims[j]] = doc.value;
}
else
{
expectedValues[j][doc.dims[j]] = (float)v + doc.value;
}
}
}
}
}
List <FacetResult> expected = new List <FacetResult>();
for (int i = 0; i < numDims; i++)
{
List <LabelAndValue> labelValues = new List <LabelAndValue>();
float totValue = 0;
foreach (KeyValuePair <string, float?> ent in expectedValues[i])
{
labelValues.Add(new LabelAndValue(ent.Key, ent.Value.Value));
totValue += ent.Value.Value;
}
SortLabelValues(labelValues);
if (totValue > 0)
{
expected.Add(new FacetResult("dim" + i, new string[0], totValue, labelValues.ToArray(), labelValues.Count));
}
}
// Sort by highest value, tie break by value:
SortFacetResults(expected);
IList <FacetResult> actual = facets.GetAllDims(10);
// Messy: fixup ties
SortTies(actual);
if (VERBOSE)
{
Console.WriteLine("expected=\n" + expected.ToString());
Console.WriteLine("actual=\n" + actual.ToString());
}
AssertFloatValuesEquals(expected, actual);
}
IOUtils.Dispose(w, tw, searcher.IndexReader, tr, indexDir, taxoDir);
}
// randomly seeks to term that we know exists, then next's
// from there
private void DoTestSeekExists(Random r, IList <Term> fieldTerms, IndexReader reader)
{
IDictionary <string, TermsEnum> tes = new Dictionary <string, TermsEnum>();
// Test random seek to existing term, then enum:
if (VERBOSE)
{
Console.WriteLine("\nTEST: top now seek");
}
int num = AtLeast(100);
for (int iter = 0; iter < num; iter++)
{
// pick random field+term
int spot = r.Next(fieldTerms.Count);
Term term = fieldTerms[spot];
string field = term.Field;
if (VERBOSE)
{
Console.WriteLine("TEST: exist seek field=" + field + " term=" + UnicodeUtil.ToHexString(term.Text()));
}
// seek to it
TermsEnum te;
if (!tes.TryGetValue(field, out te))
{
te = MultiFields.GetTerms(reader, field).GetIterator(null);
tes[field] = te;
}
if (VERBOSE)
{
Console.WriteLine(" done get enum");
}
// seek should find the term
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, te.SeekCeil(term.Bytes));
// now .next() this many times:
int ct = TestUtil.NextInt32(r, 5, 100);
for (int i = 0; i < ct; i++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: now next()");
}
if (1 + spot + i >= fieldTerms.Count)
{
break;
}
term = fieldTerms[1 + spot + i];
if (!term.Field.Equals(field, StringComparison.Ordinal))
{
Assert.IsNull(te.Next());
break;
}
else
{
BytesRef t = te.Next();
if (VERBOSE)
{
Console.WriteLine(" got term=" + (t == null ? null : UnicodeUtil.ToHexString(t.Utf8ToString())));
Console.WriteLine(" exp=" + UnicodeUtil.ToHexString(term.Text().ToString()));
}
Assert.AreEqual(term.Bytes, t);
}
}
}
}
public virtual void TestSurrogatesOrder()
{
Directory dir = NewDirectory();
var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
config.Codec = new PreFlexRWCodec();
RandomIndexWriter w = new RandomIndexWriter(Random, dir, config);
int numField = TestUtil.NextInt32(Random, 2, 5);
int uniqueTermCount = 0;
int tc = 0;
var fieldTerms = new List <Term>();
for (int f = 0; f < numField; f++)
{
string field = "f" + f;
int numTerms = AtLeast(200);
ISet <string> uniqueTerms = new JCG.HashSet <string>();
for (int i = 0; i < numTerms; i++)
{
string term = GetRandomString(Random) + "_ " + (tc++);
uniqueTerms.Add(term);
fieldTerms.Add(new Term(field, term));
Documents.Document doc = new Documents.Document();
doc.Add(NewStringField(field, term, Field.Store.NO));
w.AddDocument(doc);
}
uniqueTermCount += uniqueTerms.Count;
}
IndexReader reader = w.GetReader();
if (VERBOSE)
{
fieldTerms.Sort(TermAsUTF16Comparer);
Console.WriteLine("\nTEST: UTF16 order");
foreach (Term t in fieldTerms)
{
Console.WriteLine(" " + ToHexString(t));
}
}
// sorts in code point order:
fieldTerms.Sort();
if (VERBOSE)
{
Console.WriteLine("\nTEST: codepoint order");
foreach (Term t in fieldTerms)
{
Console.WriteLine(" " + ToHexString(t));
}
}
Term[] fieldTermsArray = fieldTerms.ToArray();
//SegmentInfo si = makePreFlexSegment(r, "_0", dir, fieldInfos, codec, fieldTerms);
//FieldsProducer fields = codec.fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, 1));
//Assert.IsNotNull(fields);
DoTestStraightEnum(fieldTerms, reader, uniqueTermCount);
DoTestSeekExists(Random, fieldTerms, reader);
DoTestSeekDoesNotExist(Random, numField, fieldTerms, fieldTermsArray, reader);
reader.Dispose();
w.Dispose();
dir.Dispose();
}
public virtual void TestIntersectRandom()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numTerms = AtLeast(300);
//final int numTerms = 50;
ISet <string> terms = new JCG.HashSet <string>();
ICollection <string> pendingTerms = new List <string>();
IDictionary <BytesRef, int?> termToID = new Dictionary <BytesRef, int?>();
int id = 0;
while (terms.Count != numTerms)
{
string s = RandomString;
if (!terms.Contains(s))
{
terms.Add(s);
pendingTerms.Add(s);
if (Random.Next(20) == 7)
{
AddDoc(w, pendingTerms, termToID, id++);
}
}
}
AddDoc(w, pendingTerms, termToID, id++);
BytesRef[] termsArray = new BytesRef[terms.Count];
ISet <BytesRef> termsSet = new JCG.HashSet <BytesRef>();
{
int upto = 0;
foreach (string s in terms)
{
BytesRef b = new BytesRef(s);
termsArray[upto++] = b;
termsSet.Add(b);
}
Array.Sort(termsArray);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: indexed terms (unicode order):");
foreach (BytesRef t in termsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + " -> id:" + termToID[t]);
}
}
IndexReader r = w.GetReader();
w.Dispose();
// NOTE: intentional insanity!!
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
{
// TODO: can we also test infinite As here...?
// From the random terms, pick some ratio and compile an
// automaton:
ISet <string> acceptTerms = new JCG.HashSet <string>();
JCG.SortedSet <BytesRef> sortedAcceptTerms = new JCG.SortedSet <BytesRef>();
double keepPct = Random.NextDouble();
Automaton a;
if (iter == 0)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: empty automaton");
}
a = BasicAutomata.MakeEmpty();
}
else
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: keepPct=" + keepPct);
}
foreach (string s in terms)
{
string s2;
if (Random.NextDouble() <= keepPct)
{
s2 = s;
}
else
{
s2 = RandomString;
}
acceptTerms.Add(s2);
sortedAcceptTerms.Add(new BytesRef(s2));
}
a = BasicAutomata.MakeStringUnion(sortedAcceptTerms);
}
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: reduce the automaton");
}
a.Reduce();
}
CompiledAutomaton c = new CompiledAutomaton(a, true, false);
BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.Count];
ISet <BytesRef> acceptTermsSet = new JCG.HashSet <BytesRef>();
int upto = 0;
foreach (string s in acceptTerms)
{
BytesRef b = new BytesRef(s);
acceptTermsArray[upto++] = b;
acceptTermsSet.Add(b);
Assert.IsTrue(Accepts(c, b));
}
Array.Sort(acceptTermsArray);
if (VERBOSE)
{
Console.WriteLine("\nTEST: accept terms (unicode order):");
foreach (BytesRef t in acceptTermsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + (termsSet.Contains(t) ? " (exists)" : ""));
}
Console.WriteLine(a.ToDot());
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
BytesRef startTerm = acceptTermsArray.Length == 0 || Random.NextBoolean() ? null : acceptTermsArray[Random.Next(acceptTermsArray.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.Utf8ToString()));
if (startTerm != null)
{
int state = c.RunAutomaton.InitialState;
for (int idx = 0; idx < startTerm.Length; idx++)
{
int label = startTerm.Bytes[startTerm.Offset + idx] & 0xff;
Console.WriteLine(" state=" + state + " label=" + label);
state = c.RunAutomaton.Step(state, label);
Assert.IsTrue(state != -1);
}
Console.WriteLine(" state=" + state);
}
}
TermsEnum te = MultiFields.GetTerms(r, "f").Intersect(c, startTerm);
int loc;
if (startTerm == null)
{
loc = 0;
}
else
{
loc = Array.BinarySearch(termsArray, BytesRef.DeepCopyOf(startTerm));
if (loc < 0)
{
loc = -(loc + 1);
}
else
{
// startTerm exists in index
loc++;
}
}
while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]))
{
loc++;
}
DocsEnum docsEnum = null;
while (loc < termsArray.Length)
{
BytesRef expected = termsArray[loc];
BytesRef actual = te.Next();
if (VERBOSE)
{
Console.WriteLine("TEST: next() expected=" + expected.Utf8ToString() + " actual=" + (actual == null ? "null" : actual.Utf8ToString()));
}
Assert.AreEqual(expected, actual);
Assert.AreEqual(1, te.DocFreq);
docsEnum = TestUtil.Docs(Random, te, null, docsEnum, DocsFlags.NONE);
int docID = docsEnum.NextDoc();
Assert.IsTrue(docID != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(docIDToID.Get(docID), (int)termToID[expected]);
do
{
loc++;
} while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]));
}
Assert.IsNull(te.Next());
}
}
r.Dispose();
dir.Dispose();
}
private void TestRandomSeeks(IndexReader r, params string[] validTermStrings)
{
BytesRef[] validTerms = new BytesRef[validTermStrings.Length];
for (int termIDX = 0; termIDX < validTermStrings.Length; termIDX++)
{
validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]);
}
Array.Sort(validTerms);
if (VERBOSE)
{
Console.WriteLine("TEST: " + validTerms.Length + " terms:");
foreach (BytesRef t in validTerms)
{
Console.WriteLine(" " + t.Utf8ToString() + " " + t);
}
}
TermsEnum te = MultiFields.GetTerms(r, FIELD).GetIterator(null);
int END_LOC = -validTerms.Length - 1;
IList <TermAndState> termStates = new List <TermAndState>();
for (int iter = 0; iter < 100 * RANDOM_MULTIPLIER; iter++)
{
BytesRef t;
int loc;
TermState termState;
if (Random.Next(6) == 4)
{
// pick term that doens't exist:
t = GetNonExistTerm(validTerms);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: invalid term=" + t.Utf8ToString());
}
loc = Array.BinarySearch(validTerms, t);
}
else if (termStates.Count != 0 && Random.Next(4) == 1)
{
TermAndState ts = termStates[Random.Next(termStates.Count)];
t = ts.Term;
loc = Array.BinarySearch(validTerms, t);
Assert.IsTrue(loc >= 0);
termState = ts.State;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid termState term=" + t.Utf8ToString());
}
}
else
{
// pick valid term
loc = Random.Next(validTerms.Length);
t = BytesRef.DeepCopyOf(validTerms[loc]);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid term=" + t.Utf8ToString());
}
}
// seekCeil or seekExact:
bool doSeekExact = Random.NextBoolean();
if (termState != null)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact termState");
}
te.SeekExact(t, termState);
}
else if (doSeekExact)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact");
}
Assert.AreEqual(loc >= 0, te.SeekExact(t));
}
else
{
if (VERBOSE)
{
Console.WriteLine(" seekCeil");
}
TermsEnum.SeekStatus result = te.SeekCeil(t);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (loc >= 0)
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, result);
}
else if (loc == END_LOC)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, result);
}
else
{
Debug.Assert(loc >= -validTerms.Length);
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, result);
}
}
if (loc >= 0)
{
Assert.AreEqual(t, te.Term);
}
else if (doSeekExact)
{
// TermsEnum is unpositioned if seekExact returns false
continue;
}
else if (loc == END_LOC)
{
continue;
}
else
{
loc = -loc - 1;
Assert.AreEqual(validTerms[loc], te.Term);
}
// Do a bunch of next's after the seek
int numNext = Random.Next(validTerms.Length);
for (int nextCount = 0; nextCount < numNext; nextCount++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: next loc=" + loc + " of " + validTerms.Length);
}
BytesRef t2 = te.Next();
loc++;
if (loc == validTerms.Length)
{
Assert.IsNull(t2);
break;
}
else
{
Assert.AreEqual(validTerms[loc], t2);
if (Random.Next(40) == 17 && termStates.Count < 100)
{
termStates.Add(new TermAndState(validTerms[loc], te.GetTermState()));
}
}
}
}
}
internal virtual void TestSort(bool useFrom, bool VERBOSE)
{
IndexReader reader = null;
Directory dir = null;
if (!VERBOSE)
{
Console.WriteLine("Verbosity disabled. Enable manually if needed.");
}
int numDocs = VERBOSE ? AtLeast(50) : AtLeast(1000);
//final int numDocs = AtLeast(50);
string[] tokens = new string[] { "a", "b", "c", "d", "e" };
if (VERBOSE)
{
Console.WriteLine("TEST: make index");
}
{
dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
// w.setDoRandomForceMerge(false);
// w.w.getConfig().SetMaxBufferedDocs(AtLeast(100));
string[] content = new string[AtLeast(20)];
for (int contentIDX = 0; contentIDX < content.Length; contentIDX++)
{
StringBuilder sb = new StringBuilder();
int numTokens = TestUtil.NextInt32(Random, 1, 10);
for (int tokenIDX = 0; tokenIDX < numTokens; tokenIDX++)
{
sb.Append(tokens[Random.Next(tokens.Length)]).Append(' ');
}
content[contentIDX] = sb.ToString();
}
for (int docIDX = 0; docIDX < numDocs; docIDX++)
{
Document doc = new Document();
doc.Add(NewStringField("string", TestUtil.RandomRealisticUnicodeString(Random), Field.Store.NO));
doc.Add(NewTextField("text", content[Random.Next(content.Length)], Field.Store.NO));
doc.Add(new SingleField("float", (float)Random.NextDouble(), Field.Store.NO));
int intValue;
if (Random.Next(100) == 17)
{
intValue = int.MinValue;
}
else if (Random.Next(100) == 17)
{
intValue = int.MaxValue;
}
else
{
intValue = Random.Next();
}
doc.Add(new Int32Field("int", intValue, Field.Store.NO));
if (VERBOSE)
{
Console.WriteLine(" doc=" + doc);
}
w.AddDocument(doc);
}
reader = w.GetReader();
w.Dispose();
}
// NOTE: sometimes reader has just one segment, which is
// important to test
IndexSearcher searcher = NewSearcher(reader);
IndexReaderContext ctx = searcher.TopReaderContext;
ShardSearcher[] subSearchers;
int[] docStarts;
if (ctx is AtomicReaderContext)
{
subSearchers = new ShardSearcher[1];
docStarts = new int[1];
subSearchers[0] = new ShardSearcher((AtomicReaderContext)ctx, ctx);
docStarts[0] = 0;
}
else
{
CompositeReaderContext compCTX = (CompositeReaderContext)ctx;
int size = compCTX.Leaves.Count;
subSearchers = new ShardSearcher[size];
docStarts = new int[size];
int docBase = 0;
for (int searcherIDX = 0; searcherIDX < subSearchers.Length; searcherIDX++)
{
AtomicReaderContext leave = compCTX.Leaves[searcherIDX];
subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX);
docStarts[searcherIDX] = docBase;
docBase += leave.Reader.MaxDoc;
}
}
IList <SortField> sortFields = new List <SortField>();
sortFields.Add(new SortField("string", SortFieldType.STRING, true));
sortFields.Add(new SortField("string", SortFieldType.STRING, false));
sortFields.Add(new SortField("int", SortFieldType.INT32, true));
sortFields.Add(new SortField("int", SortFieldType.INT32, false));
sortFields.Add(new SortField("float", SortFieldType.SINGLE, true));
sortFields.Add(new SortField("float", SortFieldType.SINGLE, false));
sortFields.Add(new SortField(null, SortFieldType.SCORE, true));
sortFields.Add(new SortField(null, SortFieldType.SCORE, false));
sortFields.Add(new SortField(null, SortFieldType.DOC, true));
sortFields.Add(new SortField(null, SortFieldType.DOC, false));
for (int iter = 0; iter < 1000 * RANDOM_MULTIPLIER; iter++)
{
// TODO: custom FieldComp...
Query query = new TermQuery(new Term("text", tokens[Random.Next(tokens.Length)]));
Sort sort;
if (Random.Next(10) == 4)
{
// Sort by score
sort = null;
}
else
{
SortField[] randomSortFields = new SortField[TestUtil.NextInt32(Random, 1, 3)];
for (int sortIDX = 0; sortIDX < randomSortFields.Length; sortIDX++)
{
randomSortFields[sortIDX] = sortFields[Random.Next(sortFields.Count)];
}
sort = new Sort(randomSortFields);
}
int numHits = TestUtil.NextInt32(Random, 1, numDocs + 5);
//final int numHits = 5;
if (VERBOSE)
{
Console.WriteLine("TEST: search query=" + query + " sort=" + sort + " numHits=" + numHits);
}
int from = -1;
int size = -1;
// First search on whole index:
TopDocs topHits;
if (sort == null)
{
if (useFrom)
{
TopScoreDocCollector c = TopScoreDocCollector.Create(numHits, Random.NextBoolean());
searcher.Search(query, c);
from = TestUtil.NextInt32(Random, 0, numHits - 1);
size = numHits - from;
TopDocs tempTopHits = c.GetTopDocs();
if (from < tempTopHits.ScoreDocs.Length)
{
// Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
// than TopDocs#merge currently has
ScoreDoc[] newScoreDocs = new ScoreDoc[Math.Min(size, tempTopHits.ScoreDocs.Length - from)];
Array.Copy(tempTopHits.ScoreDocs, from, newScoreDocs, 0, newScoreDocs.Length);
tempTopHits.ScoreDocs = newScoreDocs;
topHits = tempTopHits;
}
else
{
topHits = new TopDocs(tempTopHits.TotalHits, new ScoreDoc[0], tempTopHits.MaxScore);
}
}
else
{
topHits = searcher.Search(query, numHits);
}
}
else
{
TopFieldCollector c = TopFieldCollector.Create(sort, numHits, true, true, true, Random.NextBoolean());
searcher.Search(query, c);
if (useFrom)
{
from = TestUtil.NextInt32(Random, 0, numHits - 1);
size = numHits - from;
TopDocs tempTopHits = c.GetTopDocs();
if (from < tempTopHits.ScoreDocs.Length)
{
// Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
// than TopDocs#merge currently has
ScoreDoc[] newScoreDocs = new ScoreDoc[Math.Min(size, tempTopHits.ScoreDocs.Length - from)];
Array.Copy(tempTopHits.ScoreDocs, from, newScoreDocs, 0, newScoreDocs.Length);
tempTopHits.ScoreDocs = newScoreDocs;
topHits = tempTopHits;
}
else
{
topHits = new TopDocs(tempTopHits.TotalHits, new ScoreDoc[0], tempTopHits.MaxScore);
}
}
else
{
topHits = c.GetTopDocs(0, numHits);
}
}
if (VERBOSE)
{
if (useFrom)
{
Console.WriteLine("from=" + from + " size=" + size);
}
Console.WriteLine(" top search: " + topHits.TotalHits + " totalHits; hits=" + (topHits.ScoreDocs == null ? "null" : topHits.ScoreDocs.Length + " maxScore=" + topHits.MaxScore));
if (topHits.ScoreDocs != null)
{
for (int hitIDX = 0; hitIDX < topHits.ScoreDocs.Length; hitIDX++)
{
ScoreDoc sd = topHits.ScoreDocs[hitIDX];
Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score);
}
}
}
// ... then all shards:
Weight w = searcher.CreateNormalizedWeight(query);
TopDocs[] shardHits = new TopDocs[subSearchers.Length];
for (int shardIDX = 0; shardIDX < subSearchers.Length; shardIDX++)
{
TopDocs subHits;
ShardSearcher subSearcher = subSearchers[shardIDX];
if (sort == null)
{
subHits = subSearcher.Search(w, numHits);
}
else
{
TopFieldCollector c = TopFieldCollector.Create(sort, numHits, true, true, true, Random.NextBoolean());
subSearcher.Search(w, c);
subHits = c.GetTopDocs(0, numHits);
}
shardHits[shardIDX] = subHits;
if (VERBOSE)
{
Console.WriteLine(" shard=" + shardIDX + " " + subHits.TotalHits + " totalHits hits=" + (subHits.ScoreDocs == null ? "null" : subHits.ScoreDocs.Length.ToString()));
if (subHits.ScoreDocs != null)
{
foreach (ScoreDoc sd in subHits.ScoreDocs)
{
Console.WriteLine(" doc=" + sd.Doc + " score=" + sd.Score);
}
}
}
}
// Merge:
TopDocs mergedHits;
if (useFrom)
{
mergedHits = TopDocs.Merge(sort, from, size, shardHits);
}
else
{
mergedHits = TopDocs.Merge(sort, numHits, shardHits);
}
if (mergedHits.ScoreDocs != null)
{
// Make sure the returned shards are correct:
for (int hitIDX = 0; hitIDX < mergedHits.ScoreDocs.Length; hitIDX++)
{
ScoreDoc sd = mergedHits.ScoreDocs[hitIDX];
Assert.AreEqual(ReaderUtil.SubIndex(sd.Doc, docStarts), sd.ShardIndex, "doc=" + sd.Doc + " wrong shard");
}
}
TestUtil.AssertEquals(topHits, mergedHits);
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestRandomMaps2()
{
Random random = Random();
int numIterations = AtLeast(3);
for (int iter = 0; iter < numIterations; iter++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST iter=" + iter);
}
char endLetter = (char)TestUtil.NextInt(random, 'b', 'z');
IDictionary <string, string> map = new Dictionary <string, string>();
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
int numMappings = AtLeast(5);
if (VERBOSE)
{
Console.WriteLine(" mappings:");
}
while (map.Count < numMappings)
{
string key = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, 7);
if (key.Length != 0 && !map.ContainsKey(key))
{
string value = TestUtil.RandomSimpleString(random);
map[key] = value;
builder.Add(key, value);
if (VERBOSE)
{
Console.WriteLine(" " + key + " -> " + value);
}
}
}
NormalizeCharMap charMap = builder.Build();
if (VERBOSE)
{
Console.WriteLine(" test random documents...");
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
string content = TestUtil.RandomSimpleStringRange(random, 'a', endLetter, AtLeast(1000));
if (VERBOSE)
{
Console.WriteLine(" content=" + content);
}
// Do stupid dog-slow mapping:
// Output string:
StringBuilder output = new StringBuilder();
// Maps output offset to input offset:
IList <int?> inputOffsets = new List <int?>();
int cumDiff = 0;
int charIdx = 0;
while (charIdx < content.Length)
{
int matchLen = -1;
string matchRepl = null;
foreach (KeyValuePair <string, string> ent in map)
{
string match = ent.Key;
if (charIdx + match.Length <= content.Length)
{
int limit = charIdx + match.Length;
bool matches = true;
for (int charIdx2 = charIdx; charIdx2 < limit; charIdx2++)
{
if (match[charIdx2 - charIdx] != content[charIdx2])
{
matches = false;
break;
}
}
if (matches)
{
string repl = ent.Value;
if (match.Length > matchLen)
{
// Greedy: longer match wins
matchLen = match.Length;
matchRepl = repl;
}
}
}
}
if (matchLen != -1)
{
// We found a match here!
if (VERBOSE)
{
Console.WriteLine(" match=" + content.Substring(charIdx, matchLen) + " @ off=" + charIdx + " repl=" + matchRepl);
}
output.Append(matchRepl);
int minLen = Math.Min(matchLen, matchRepl.Length);
// Common part, directly maps back to input
// offset:
for (int outIdx = 0; outIdx < minLen; outIdx++)
{
inputOffsets.Add(output.Length - matchRepl.Length + outIdx + cumDiff);
}
cumDiff += matchLen - matchRepl.Length;
charIdx += matchLen;
if (matchRepl.Length < matchLen)
{
// Replacement string is shorter than matched
// input: nothing to do
}
else if (matchRepl.Length > matchLen)
{
// Replacement string is longer than matched
// input: for all the "extra" chars we map
// back to a single input offset:
for (int outIdx = matchLen; outIdx < matchRepl.Length; outIdx++)
{
inputOffsets.Add(output.Length + cumDiff - 1);
}
}
else
{
// Same length: no change to offset
}
Debug.Assert(inputOffsets.Count == output.Length, "inputOffsets.size()=" + inputOffsets.Count + " vs output.length()=" + output.Length);
}
else
{
inputOffsets.Add(output.Length + cumDiff);
output.Append(content[charIdx]);
charIdx++;
}
}
string expected = output.ToString();
if (VERBOSE)
{
Console.Write(" expected:");
for (int charIdx2 = 0; charIdx2 < expected.Length; charIdx2++)
{
Console.Write(" " + expected[charIdx2] + "/" + inputOffsets[charIdx2]);
}
Console.WriteLine();
}
MappingCharFilter mapFilter = new MappingCharFilter(charMap, new StringReader(content));
StringBuilder actualBuilder = new StringBuilder();
IList <int?> actualInputOffsets = new List <int?>();
// Now consume the actual mapFilter, somewhat randomly:
while (true)
{
if (random.Next(0, 1) == 1)
{
int ch = mapFilter.Read();
if (ch == -1)
{
break;
}
actualBuilder.Append((char)ch);
}
else
{
char[] buffer = new char[TestUtil.NextInt(random, 1, 100)];
int off = buffer.Length == 1 ? 0 : random.Next(buffer.Length - 1);
int count = mapFilter.Read(buffer, off, buffer.Length - off);
if (count == -1)
{
break;
}
else
{
actualBuilder.Append(buffer, off, count);
}
}
if (random.Next(10) == 7)
{
// Map offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
}
}
// Finish mappping offsets
while (actualInputOffsets.Count < actualBuilder.Length)
{
actualInputOffsets.Add(mapFilter.CorrectOffset(actualInputOffsets.Count));
}
string actual = actualBuilder.ToString();
// Verify:
assertEquals(expected, actual);
assertEquals(inputOffsets, actualInputOffsets);
}
}
}
public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
{
int minSkipInterval;
if (state.SegmentInfo.DocCount > 1000000)
{
// Test2BPostings can OOME otherwise:
minSkipInterval = 3;
}
else
{
minSkipInterval = 2;
}
// we pull this before the seed intentionally: because its not consumed at runtime
// (the skipInterval is written into postings header)
int skipInterval = TestUtil.NextInt(seedRandom, minSkipInterval, 10);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: skipInterval=" + skipInterval);
}
long seed = seedRandom.nextLong();
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: writing to seg=" + state.SegmentInfo.Name + " formatID=" + state.SegmentSuffix + " seed=" + seed);
}
string seedFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, SEED_EXT);
IndexOutput @out = state.Directory.CreateOutput(seedFileName, state.Context);
try
{
@out.WriteInt64(seed);
}
finally
{
@out.Dispose();
}
Random random = new Random((int)seed);
random.nextInt(); // consume a random for buffersize
PostingsWriterBase postingsWriter;
if (random.nextBoolean())
{
postingsWriter = new SepPostingsWriter(state, new MockInt32StreamFactory(random), skipInterval);
}
else
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: writing Standard postings");
}
// TODO: randomize variables like acceptibleOverHead?!
postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
}
if (random.nextBoolean())
{
int totTFCutoff = TestUtil.NextInt(random, 1, 20);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff);
}
postingsWriter = new PulsingPostingsWriter(state, totTFCutoff, postingsWriter);
}
FieldsConsumer fields;
int t1 = random.nextInt(4);
if (t1 == 0)
{
bool success = false;
try
{
fields = new FSTTermsWriter(state, postingsWriter);
success = true;
}
finally
{
if (!success)
{
postingsWriter.Dispose();
}
}
}
else if (t1 == 1)
{
bool success = false;
try
{
fields = new FSTOrdTermsWriter(state, postingsWriter);
success = true;
}
finally
{
if (!success)
{
postingsWriter.Dispose();
}
}
}
else if (t1 == 2)
{
// Use BlockTree terms dict
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: writing BlockTree terms dict");
}
// TODO: would be nice to allow 1 but this is very
// slow to write
int minTermsInBlock = TestUtil.NextInt(random, 2, 100);
int maxTermsInBlock = Math.Max(2, (minTermsInBlock - 1) * 2 + random.nextInt(100));
bool success = false;
try
{
fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
success = true;
}
finally
{
if (!success)
{
postingsWriter.Dispose();
}
}
}
else
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: writing Block terms dict");
}
bool success = false;
TermsIndexWriterBase indexWriter;
try
{
if (random.nextBoolean())
{
state.TermIndexInterval = TestUtil.NextInt(random, 1, 100);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: fixed-gap terms index (tii=" + state.TermIndexInterval + ")");
}
indexWriter = new FixedGapTermsIndexWriter(state);
}
else
{
VariableGapTermsIndexWriter.IndexTermSelector selector;
int n2 = random.nextInt(3);
if (n2 == 0)
{
int tii = TestUtil.NextInt(random, 1, 100);
selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: variable-gap terms index (tii=" + tii + ")");
}
}
else if (n2 == 1)
{
int docFreqThresh = TestUtil.NextInt(random, 2, 100);
int tii = TestUtil.NextInt(random, 1, 100);
selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii);
}
else
{
int seed2 = random.Next();
int gap = TestUtil.NextInt(random, 2, 40);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: random-gap terms index (max gap=" + gap + ")");
}
selector = new IndexTermSelectorAnonymousHelper(seed2, gap);
// selector = new VariableGapTermsIndexWriter.IndexTermSelector() {
// Random rand = new Random(seed2);
//@Override
// public bool isIndexTerm(BytesRef term, TermStats stats)
//{
// return rand.nextInt(gap) == gap / 2;
//}
//@Override
// public void newField(FieldInfo fieldInfo)
//{
//}
// };
}
indexWriter = new VariableGapTermsIndexWriter(state, selector);
}
success = true;
}
finally
{
if (!success)
{
postingsWriter.Dispose();
}
}
success = false;
try
{
fields = new BlockTermsWriter(indexWriter, state, postingsWriter);
success = true;
}
finally
{
if (!success)
{
try
{
postingsWriter.Dispose();
}
finally
{
indexWriter.Dispose();
}
}
}
}
return(fields);
}
public override FieldsProducer FieldsProducer(SegmentReadState state)
{
string seedFileName = IndexFileNames.SegmentFileName(state.SegmentInfo.Name, state.SegmentSuffix, SEED_EXT);
IndexInput @in = state.Directory.OpenInput(seedFileName, state.Context);
long seed = @in.ReadInt64();
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading from seg=" + state.SegmentInfo.Name + " formatID=" + state.SegmentSuffix + " seed=" + seed);
}
@in.Dispose();
Random random = new Random((int)seed);
int readBufferSize = TestUtil.NextInt(random, 1, 4096);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: readBufferSize=" + readBufferSize);
}
PostingsReaderBase postingsReader;
if (random.nextBoolean())
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading Sep postings");
}
postingsReader = new SepPostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo,
state.Context, new MockInt32StreamFactory(random), state.SegmentSuffix);
}
else
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading Standard postings");
}
postingsReader = new Lucene41PostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo, state.Context, state.SegmentSuffix);
}
if (random.nextBoolean())
{
int totTFCutoff = TestUtil.NextInt(random, 1, 20);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading pulsing postings with totTFCutoff=" + totTFCutoff);
}
postingsReader = new PulsingPostingsReader(state, postingsReader);
}
FieldsProducer fields;
int t1 = random.nextInt(4);
if (t1 == 0)
{
bool success = false;
try
{
fields = new FSTTermsReader(state, postingsReader);
success = true;
}
finally
{
if (!success)
{
postingsReader.Dispose();
}
}
}
else if (t1 == 1)
{
bool success = false;
try
{
fields = new FSTOrdTermsReader(state, postingsReader);
success = true;
}
finally
{
if (!success)
{
postingsReader.Dispose();
}
}
}
else if (t1 == 2)
{
// Use BlockTree terms dict
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading BlockTree terms dict");
}
bool success = false;
try
{
fields = new BlockTreeTermsReader(state.Directory,
state.FieldInfos,
state.SegmentInfo,
postingsReader,
state.Context,
state.SegmentSuffix,
state.TermsIndexDivisor);
success = true;
}
finally
{
if (!success)
{
postingsReader.Dispose();
}
}
}
else
{
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: reading Block terms dict");
}
TermsIndexReaderBase indexReader;
bool success = false;
try
{
bool doFixedGap = random.nextBoolean();
// randomness diverges from writer, here:
if (state.TermsIndexDivisor != -1)
{
state.TermsIndexDivisor = TestUtil.NextInt(random, 1, 10);
}
if (doFixedGap)
{
// if termsIndexDivisor is set to -1, we should not touch it. It means a
// test explicitly instructed not to load the terms index.
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: fixed-gap terms index (divisor=" + state.TermsIndexDivisor + ")");
}
indexReader = new FixedGapTermsIndexReader(state.Directory,
state.FieldInfos,
state.SegmentInfo.Name,
state.TermsIndexDivisor,
BytesRef.UTF8SortedAsUnicodeComparer,
state.SegmentSuffix, state.Context);
}
else
{
int n2 = random.nextInt(3);
if (n2 == 1)
{
random.nextInt();
}
else if (n2 == 2)
{
random.nextLong();
}
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("MockRandomCodec: variable-gap terms index (divisor=" + state.TermsIndexDivisor + ")");
}
indexReader = new VariableGapTermsIndexReader(state.Directory,
state.FieldInfos,
state.SegmentInfo.Name,
state.TermsIndexDivisor,
state.SegmentSuffix, state.Context);
}
success = true;
}
finally
{
if (!success)
{
postingsReader.Dispose();
}
}
success = false;
try
{
fields = new BlockTermsReader(indexReader,
state.Directory,
state.FieldInfos,
state.SegmentInfo,
postingsReader,
state.Context,
state.SegmentSuffix);
success = true;
}
finally
{
if (!success)
{
try
{
postingsReader.Dispose();
}
finally
{
indexReader.Dispose();
}
}
}
}
return(fields);
}
public virtual void TestStressAdvance_Mem()
{
for (int iter = 0; iter < 3; iter++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter);
}
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
HashSet <int> aDocs = new HashSet <int>();
Documents.Document doc = new Documents.Document();
Field f = NewStringField("field", "", Field.Store.NO);
doc.Add(f);
Field idField = NewStringField("id", "", Field.Store.YES);
doc.Add(idField);
int num = AtLeast(4097);
if (VERBOSE)
{
Console.WriteLine("\nTEST: numDocs=" + num);
}
for (int id = 0; id < num; id++)
{
if (Random().Next(4) == 3)
{
f.SetStringValue("a");
aDocs.Add(id);
}
else
{
f.SetStringValue("b");
}
idField.SetStringValue("" + id);
w.AddDocument(doc);
if (VERBOSE)
{
Console.WriteLine("\nTEST: doc upto " + id);
}
}
w.ForceMerge(1);
IList <int> aDocIDs = new List <int>();
IList <int> bDocIDs = new List <int>();
DirectoryReader r = w.Reader;
int[] idToDocID = new int[r.MaxDoc];
for (int docID = 0; docID < idToDocID.Length; docID++)
{
int id = Convert.ToInt32(r.Document(docID).Get("id"));
if (aDocs.Contains(id))
{
aDocIDs.Add(docID);
}
else
{
bDocIDs.Add(docID);
}
}
TermsEnum te = GetOnlySegmentReader(r).Fields.GetTerms("field").GetIterator(null);
DocsEnum de = null;
for (int iter2 = 0; iter2 < 10; iter2++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter + " iter2=" + iter2);
}
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, te.SeekCeil(new BytesRef("a")));
de = TestUtil.Docs(Random(), te, null, de, DocsFlags.NONE);
TestOne(de, aDocIDs);
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, te.SeekCeil(new BytesRef("b")));
de = TestUtil.Docs(Random(), te, null, de, DocsFlags.NONE);
TestOne(de, bDocIDs);
}
w.Dispose();
r.Dispose();
dir.Dispose();
}
}
public virtual void TestPerFieldCodec()
{
int NUM_DOCS = AtLeast(173);
if (VERBOSE)
{
Console.WriteLine("TEST: NUM_DOCS=" + NUM_DOCS);
}
using (BaseDirectoryWrapper dir = NewDirectory())
{
dir.CheckIndexOnDispose = false; // we use a custom codec provider
using (IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetCodec(new CustomPerFieldCodec()).SetMergePolicy(NewLogMergePolicy(3))))
{
Documents.Document doc = new Documents.Document();
// uses default codec:
doc.Add(NewTextField("field1", "this field uses the standard codec as the test", Field.Store.NO));
// uses pulsing codec:
Field field2 = NewTextField("field2", "this field uses the pulsing codec as the test", Field.Store.NO);
doc.Add(field2);
Field idField = NewStringField("id", "", Field.Store.NO);
doc.Add(idField);
for (int i = 0; i < NUM_DOCS; i++)
{
idField.SetStringValue("" + i);
w.AddDocument(doc);
if ((i + 1) % 10 == 0)
{
w.Commit();
}
}
if (VERBOSE)
{
Console.WriteLine("TEST: now delete id=77");
}
w.DeleteDocuments(new Term("id", "77"));
using (IndexReader r = DirectoryReader.Open(w, true))
{
Assert.AreEqual(NUM_DOCS - 1, r.NumDocs);
IndexSearcher s = NewSearcher(r);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 1, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: now delete 2nd doc");
}
w.DeleteDocuments(new Term("id", "44"));
if (VERBOSE)
{
Console.WriteLine("\nTEST: now force merge");
}
w.ForceMerge(1);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now open reader");
}
using (IndexReader r = DirectoryReader.Open(w, true))
{
Assert.AreEqual(NUM_DOCS - 2, r.MaxDoc);
Assert.AreEqual(NUM_DOCS - 2, r.NumDocs);
IndexSearcher s = NewSearcher(r);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field1", "standard")), 1).TotalHits);
Assert.AreEqual(NUM_DOCS - 2, s.Search(new TermQuery(new Term("field2", "pulsing")), 1).TotalHits);
Assert.AreEqual(1, s.Search(new TermQuery(new Term("id", "76")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "77")), 1).TotalHits);
Assert.AreEqual(0, s.Search(new TermQuery(new Term("id", "44")), 1).TotalHits);
if (VERBOSE)
{
Console.WriteLine("\nTEST: now close NRT reader");
}
}
}
}
}
/*
* public void testWikipedia() {
* final FileInputStream fis = new FileInputStream("/q/lucene/jawiki-20120220-pages-articles.xml");
* final Reader r = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8));
*
* final long startTimeNS = System.nanoTime();
* boolean done = false;
* long compoundCount = 0;
* long nonCompoundCount = 0;
* long netOffset = 0;
* while (!done) {
* final TokenStream ts = analyzer.tokenStream("ignored", r);
* ts.reset();
* final PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
* final OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
* int count = 0;
* while (true) {
* if (!ts.incrementToken()) {
* done = true;
* break;
* }
* count++;
* if (posIncAtt.getPositionIncrement() == 0) {
* compoundCount++;
* } else {
* nonCompoundCount++;
* if (nonCompoundCount % 1000000 == 0) {
* System.out.println(String.format("%.2f msec [pos=%d, %d, %d]",
* (System.nanoTime()-startTimeNS)/1000000.0,
* netOffset + offsetAtt.startOffset(),
* nonCompoundCount,
* compoundCount));
* }
* }
* if (count == 100000000) {
* System.out.println(" again...");
* break;
* }
* }
* ts.end();
* netOffset += offsetAtt.endOffset();
* }
* System.out.println("compoundCount=" + compoundCount + " nonCompoundCount=" + nonCompoundCount);
* r.close();
* }
*/
private void doTestBocchan(int numIterations)
{
TextReader reader = new StreamReader(
this.GetType().getResourceAsStream("bocchan.utf-8"), Encoding.UTF8);
String line = reader.ReadLine();
reader.Dispose();
if (VERBOSE)
{
Console.WriteLine("Test for Bocchan without pre-splitting sentences");
}
/*
* if (numIterations > 1) {
* // warmup
* for (int i = 0; i < numIterations; i++) {
* final TokenStream ts = analyzer.tokenStream("ignored", line);
* ts.reset();
* while(ts.incrementToken());
* }
* }
*/
long totalStart = Environment.TickCount;
for (int i = 0; i < numIterations; i++)
{
TokenStream ts = analyzer.GetTokenStream("ignored", line);
try
{
ts.Reset();
while (ts.IncrementToken())
{
;
}
ts.End();
}
finally
{
IOUtils.DisposeWhileHandlingException(ts);
}
}
String[] sentences = Regex.Split(line, "、|。").TrimEnd();
if (VERBOSE)
{
Console.WriteLine("Total time : " + (Environment.TickCount - totalStart));
Console.WriteLine("Test for Bocchan with pre-splitting sentences (" + sentences.Length + " sentences)");
}
totalStart = Environment.TickCount;
for (int i = 0; i < numIterations; i++)
{
foreach (String sentence in sentences)
{
TokenStream ts = analyzer.GetTokenStream("ignored", sentence);
try
{
ts.Reset();
while (ts.IncrementToken())
{
;
}
ts.End();
}
finally
{
IOUtils.DisposeWhileHandlingException(ts);
}
}
}
if (VERBOSE)
{
Console.WriteLine("Total time : " + (Environment.TickCount - totalStart));
}
}
public virtual void TestChangeCodecAndMerge()
{
Directory dir = NewDirectory();
if (VERBOSE)
{
Console.WriteLine("TEST: make new index");
}
IndexWriterConfig iwconf = NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)).SetOpenMode(OpenMode.CREATE).SetCodec(new MockCodec());
iwconf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
// ((LogMergePolicy)iwconf.getMergePolicy()).setMergeFactor(10);
IndexWriter writer = NewWriter(dir, iwconf);
AddDocs(writer, 10);
writer.Commit();
AssertQuery(new Term("content", "aaa"), dir, 10);
if (VERBOSE)
{
Console.WriteLine("TEST: addDocs3");
}
AddDocs3(writer, 10);
writer.Commit();
writer.Dispose();
AssertQuery(new Term("content", "ccc"), dir, 10);
AssertQuery(new Term("content", "aaa"), dir, 10);
Codec codec = iwconf.Codec;
iwconf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetOpenMode(OpenMode.APPEND).SetCodec(codec);
// ((LogMergePolicy)iwconf.getMergePolicy()).setNoCFSRatio(0.0);
// ((LogMergePolicy)iwconf.getMergePolicy()).setMergeFactor(10);
iwconf.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwconf.SetCodec(new MockCodec2()); // uses standard for field content
writer = NewWriter(dir, iwconf);
// swap in new codec for currently written segments
if (VERBOSE)
{
Console.WriteLine("TEST: add docs w/ Standard codec for content field");
}
AddDocs2(writer, 10);
writer.Commit();
codec = iwconf.Codec;
Assert.AreEqual(30, writer.MaxDoc);
AssertQuery(new Term("content", "bbb"), dir, 10);
AssertQuery(new Term("content", "ccc"), dir, 10); ////
AssertQuery(new Term("content", "aaa"), dir, 10);
if (VERBOSE)
{
Console.WriteLine("TEST: add more docs w/ new codec");
}
AddDocs2(writer, 10);
writer.Commit();
AssertQuery(new Term("content", "ccc"), dir, 10);
AssertQuery(new Term("content", "bbb"), dir, 20);
AssertQuery(new Term("content", "aaa"), dir, 10);
Assert.AreEqual(40, writer.MaxDoc);
if (VERBOSE)
{
Console.WriteLine("TEST: now optimize");
}
writer.ForceMerge(1);
Assert.AreEqual(40, writer.MaxDoc);
writer.Dispose();
AssertQuery(new Term("content", "ccc"), dir, 10);
AssertQuery(new Term("content", "bbb"), dir, 20);
AssertQuery(new Term("content", "aaa"), dir, 10);
dir.Dispose();
}
public override int DoLogic()
{
int res = 0;
// open reader or use existing one
IndexSearcher searcher = RunData.GetIndexSearcher();
IndexReader reader;
bool closeSearcher;
if (searcher == null)
{
// open our own reader
Directory dir = RunData.Directory;
reader = DirectoryReader.Open(dir);
searcher = new IndexSearcher(reader);
closeSearcher = true;
}
else
{
// use existing one; this passes +1 ref to us
reader = searcher.IndexReader;
closeSearcher = false;
}
// optionally warm and add num docs traversed to count
if (WithWarm)
{
Document doc = null;
IBits liveDocs = MultiFields.GetLiveDocs(reader);
for (int m = 0; m < reader.MaxDoc; m++)
{
if (null == liveDocs || liveDocs.Get(m))
{
doc = reader.Document(m);
res += (doc == null ? 0 : 1);
}
}
}
if (WithSearch)
{
res++;
Query q = queryMaker.MakeQuery();
Sort sort = Sort;
TopDocs hits = null;
int numHits = NumHits;
if (numHits > 0)
{
if (WithCollector == false)
{
if (sort != null)
{
// TODO: instead of always passing false we
// should detect based on the query; if we make
// the IndexSearcher search methods that take
// Weight public again, we can go back to
// pulling the Weight ourselves:
TopFieldCollector collector = TopFieldCollector.Create(sort, numHits,
true, WithScore,
WithMaxScore,
false);
searcher.Search(q, null, collector);
hits = collector.GetTopDocs();
}
else
{
hits = searcher.Search(q, numHits);
}
}
else
{
ICollector collector = CreateCollector();
searcher.Search(q, null, collector);
//hits = collector.topDocs();
}
string printHitsField = RunData.Config.Get("print.hits.field", null);
if (hits != null && printHitsField != null && printHitsField.Length > 0)
{
Console.WriteLine("totalHits = " + hits.TotalHits);
Console.WriteLine("maxDoc() = " + reader.MaxDoc);
Console.WriteLine("numDocs() = " + reader.NumDocs);
for (int i = 0; i < hits.ScoreDocs.Length; i++)
{
int docID = hits.ScoreDocs[i].Doc;
Document doc = reader.Document(docID);
Console.WriteLine(" " + i + ": doc=" + docID + " score=" + hits.ScoreDocs[i].Score + " " + printHitsField + " =" + doc.Get(printHitsField));
}
}
if (WithTraverse)
{
ScoreDoc[] scoreDocs = hits.ScoreDocs;
int traversalSize = Math.Min(scoreDocs.Length, TraversalSize);
if (traversalSize > 0)
{
bool retrieve = WithRetrieve;
int numHighlight = Math.Min(NumToHighlight, scoreDocs.Length);
Analyzer analyzer = RunData.Analyzer;
BenchmarkHighlighter highlighter = null;
if (numHighlight > 0)
{
highlighter = GetBenchmarkHighlighter(q);
}
for (int m = 0; m < traversalSize; m++)
{
int id = scoreDocs[m].Doc;
res++;
if (retrieve)
{
Document document = RetrieveDoc(reader, id);
res += document != null ? 1 : 0;
if (numHighlight > 0 && m < numHighlight)
{
ICollection <string> fieldsToHighlight = GetFieldsToHighlight(document);
foreach (string field in fieldsToHighlight)
{
string text = document.Get(field);
res += highlighter.DoHighlight(reader, id, field, document, analyzer, text);
}
}
}
}
}
}
}
}
if (closeSearcher)
{
reader.Dispose();
}
else
{
// Release our +1 ref from above
reader.DecRef();
}
return(res);
}
public static void Main(string[] args)
{
if (args.Length != 7)
{
// LUCENENET specific - our wrapper console shows the correct usage
throw new ArgumentException();
//Console.WriteLine("Usage: java Lucene.Net.Store.LockStressTest myID verifierHost verifierPort lockFactoryClassName lockDirName sleepTimeMS count\n" +
// "\n" +
// " myID = int from 0 .. 255 (should be unique for test process)\n" +
// " verifierHost = hostname that LockVerifyServer is listening on\n" +
// " verifierPort = port that LockVerifyServer is listening on\n" +
// " lockFactoryClassName = primary LockFactory class that we will use\n" +
// " lockDirName = path to the lock directory (only set for Simple/NativeFSLockFactory\n" +
// " sleepTimeMS = milliseconds to pause betweeen each lock obtain/release\n" +
// " count = number of locking tries\n" +
// "\n" +
// "You should run multiple instances of this process, each with its own\n" +
// "unique ID, and each pointing to the same lock directory, to verify\n" +
// "that locking is working correctly.\n" +
// "\n" +
// "Make sure you are first running LockVerifyServer.");
//Environment.FailFast("1");
}
int arg = 0;
int myID = Convert.ToInt32(args[arg++], CultureInfo.InvariantCulture);
if (myID < 0 || myID > 255)
{
throw new ArgumentException("ID must be a unique int 0..255");
//Console.WriteLine("myID must be a unique int 0..255");
//Environment.Exit(1);
}
string verifierHost = args[arg++];
int verifierPort = Convert.ToInt32(args[arg++], CultureInfo.InvariantCulture);
string lockFactoryClassName = args[arg++];
string lockDirName = args[arg++];
int sleepTimeMS = Convert.ToInt32(args[arg++], CultureInfo.InvariantCulture);
int count = Convert.ToInt32(args[arg++], CultureInfo.InvariantCulture);
IPAddress[] addresses = Dns.GetHostAddressesAsync(verifierHost).Result;
IPAddress addr = addresses.FirstOrDefault();
Type c;
try
{
c = Type.GetType(lockFactoryClassName);
if (c == null)
{
// LUCENENET: try again, this time with the Store namespace
c = Type.GetType("Lucene.Net.Store." + lockFactoryClassName);
}
}
catch (Exception)
{
throw new IOException("unable to find LockClass " + lockFactoryClassName);
}
LockFactory lockFactory;
try
{
lockFactory = (LockFactory)Activator.CreateInstance(c);
}
catch (UnauthorizedAccessException e)
{
throw new IOException("Cannot instantiate lock factory " + lockFactoryClassName, e);
}
catch (InvalidCastException e)
{
throw new IOException("unable to cast LockClass " + lockFactoryClassName + " instance to a LockFactory", e);
}
catch (Exception e)
{
throw new IOException("InstantiationException when instantiating LockClass " + lockFactoryClassName, e);
}
DirectoryInfo lockDir = new DirectoryInfo(lockDirName);
if (lockFactory is FSLockFactory)
{
((FSLockFactory)lockFactory).SetLockDir(lockDir);
}
Console.WriteLine("Connecting to server " + addr + " and registering as client " + myID + "...");
using (Socket socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp))
{
socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, 1);
socket.Connect(verifierHost, verifierPort);
using (Stream stream = new NetworkStream(socket))
{
BinaryReader intReader = new BinaryReader(stream);
BinaryWriter intWriter = new BinaryWriter(stream);
intWriter.Write(myID);
stream.Flush();
lockFactory.LockPrefix = "test";
LockFactory verifyLF = new VerifyingLockFactory(lockFactory, stream);
Lock l = verifyLF.MakeLock("test.lock");
Random rnd = new Random();
// wait for starting gun
if (intReader.ReadInt32() != 43)
{
throw new IOException("Protocol violation");
}
for (int i = 0; i < count; i++)
{
bool obtained = false;
try
{
obtained = l.Obtain(rnd.Next(100) + 10);
}
#pragma warning disable 168
catch (LockObtainFailedException e)
#pragma warning restore 168
{
}
if (obtained)
{
Thread.Sleep(sleepTimeMS);
l.Dispose();
}
if (i % 500 == 0)
{
Console.WriteLine((i * 100.0 / count) + "% done.");
}
Thread.Sleep(sleepTimeMS);
}
}
}
Console.WriteLine("Finished " + count + " tries.");
}
public virtual void TestKeepAllDeletionPolicy()
{
for (int pass = 0; pass < 2; pass++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: cycle pass="******"TEST: open writer for forceMerge");
}
writer = new IndexWriter(dir, conf);
policy = (KeepAllDeletionPolicy)writer.Config.IndexDeletionPolicy;
writer.ForceMerge(1);
writer.Dispose();
}
Assert.AreEqual(needsMerging ? 2 : 1, policy.NumOnInit);
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
Assert.AreEqual(1 + (needsMerging ? 1 : 0), policy.NumOnCommit);
// Test listCommits
ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir);
// 2 from closing writer
Assert.AreEqual(1 + (needsMerging ? 1 : 0), commits.Count);
// Make sure we can open a reader on each commit:
foreach (IndexCommit commit in commits)
{
IndexReader r = DirectoryReader.Open(commit);
r.Dispose();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
dir.DeleteFile(IndexFileNames.SEGMENTS_GEN);
long gen = SegmentInfos.GetLastCommitGeneration(dir);
while (gen > 0)
{
IndexReader reader = DirectoryReader.Open(dir);
reader.Dispose();
dir.DeleteFile(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0)
{
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.ListAll().Length;
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetOpenMode(OpenMode.APPEND).SetIndexDeletionPolicy(policy));
writer.Dispose();
int postCount = dir.ListAll().Length;
Assert.IsTrue(postCount < preCount);
}
}
dir.Dispose();
}
}
public virtual void Test()
{
Random random = new Random(Random.Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues);
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
LuceneTestCase.Random, d, analyzer);
int numDocs = AtLeast(10);
for (int docCount = 0; docCount < numDocs; docCount++)
{
w.AddDocument(docs.NextDoc());
}
IndexReader r = w.GetReader();
w.Dispose();
List <BytesRef> terms = new List <BytesRef>();
TermsEnum termsEnum = MultiFields.GetTerms(r, "body").GetIterator(null);
BytesRef term;
while ((term = termsEnum.Next()) != null)
{
terms.Add(BytesRef.DeepCopyOf(term));
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + terms.Count + " terms");
}
int upto = -1;
int iters = AtLeast(200);
for (int iter = 0; iter < iters; iter++)
{
bool isEnd;
if (upto != -1 && LuceneTestCase.Random.NextBoolean())
{
// next
if (VERBOSE)
{
Console.WriteLine("TEST: iter next");
}
isEnd = termsEnum.Next() == null;
upto++;
if (isEnd)
{
if (VERBOSE)
{
Console.WriteLine(" end");
}
Assert.AreEqual(upto, terms.Count);
upto = -1;
}
else
{
if (VERBOSE)
{
Console.WriteLine(" got term=" + termsEnum.Term.Utf8ToString() + " expected=" + terms[upto].Utf8ToString());
}
Assert.IsTrue(upto < terms.Count);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
BytesRef target;
string exists;
if (LuceneTestCase.Random.NextBoolean())
{
// likely fake term
if (LuceneTestCase.Random.NextBoolean())
{
target = new BytesRef(TestUtil.RandomSimpleString(LuceneTestCase.Random));
}
else
{
target = new BytesRef(TestUtil.RandomRealisticUnicodeString(LuceneTestCase.Random));
}
exists = "likely not";
}
else
{
// real term
target = terms[LuceneTestCase.Random.Next(terms.Count)];
exists = "yes";
}
upto = terms.BinarySearch(target);
if (LuceneTestCase.Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekCeil target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekCeil
TermsEnum.SeekStatus status = termsEnum.SeekCeil(target);
if (VERBOSE)
{
Console.WriteLine(" got " + status);
}
if (upto < 0)
{
upto = -(upto + 1);
if (upto >= terms.Count)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, status);
upto = -1;
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekExact target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekExact
bool result = termsEnum.SeekExact(target);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (upto < 0)
{
Assert.IsFalse(result);
upto = -1;
}
else
{
Assert.IsTrue(result);
Assert.AreEqual(target, termsEnum.Term);
}
}
}
}
r.Dispose();
d.Dispose();
docs.Dispose();
}
public void assertFromTestData(int[] codePointTable)
{
if (VERBOSE)
{
Console.WriteLine("TEST: codePointTable=" + codePointTable);
}
Stream stream = GetType().getResourceAsStream("fuzzyTestData.txt");
TextReader reader = new StreamReader(stream, Encoding.UTF8);
int bits = int.Parse(reader.ReadLine(), CultureInfo.InvariantCulture);
int terms = (int)Math.Pow(2, bits);
Store.Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random, MockTokenizer.KEYWORD, false)).SetMergePolicy(NewLogMergePolicy()));
Document doc = new Document();
Field field = NewTextField("field", "", Field.Store.NO);
doc.Add(field);
for (int i = 0; i < terms; i++)
{
field.SetStringValue(MapInt(codePointTable, i));
writer.AddDocument(doc);
}
IndexReader r = writer.GetReader();
IndexSearcher searcher = NewSearcher(r);
if (VERBOSE)
{
Console.WriteLine("TEST: searcher=" + searcher);
}
// even though this uses a boost-only rewrite, this test relies upon queryNorm being the default implementation,
// otherwise scores are different!
searcher.Similarity = (new DefaultSimilarity());
writer.Dispose();
String line;
int lineNum = 0;
while ((line = reader.ReadLine()) != null)
{
lineNum++;
String[] @params = line.Split(',').TrimEnd();
String query = MapInt(codePointTable, int.Parse(@params[0], CultureInfo.InvariantCulture));
int prefix = int.Parse(@params[1], CultureInfo.InvariantCulture);
int pqSize = int.Parse(@params[2], CultureInfo.InvariantCulture);
float minScore = float.Parse(@params[3], CultureInfo.InvariantCulture);
#pragma warning disable 612, 618
SlowFuzzyQuery q = new SlowFuzzyQuery(new Term("field", query), minScore, prefix);
#pragma warning restore 612, 618
q.MultiTermRewriteMethod = new MultiTermQuery.TopTermsBoostOnlyBooleanQueryRewrite(pqSize);
int expectedResults = int.Parse(reader.ReadLine(), CultureInfo.InvariantCulture);
TopDocs docs = searcher.Search(q, expectedResults);
assertEquals(expectedResults, docs.TotalHits);
for (int i = 0; i < expectedResults; i++)
{
String[] scoreDoc = reader.ReadLine().Split(',').TrimEnd();
assertEquals(int.Parse(scoreDoc[0], CultureInfo.InvariantCulture), docs.ScoreDocs[i].Doc);
assertEquals(float.Parse(scoreDoc[1], CultureInfo.InvariantCulture), docs.ScoreDocs[i].Score, epsilon);
}
}
r.Dispose();
dir.Dispose();
}
private void TestRandomTrieAndClassicRangeQuery(int precisionStep)
{
string field = "field" + precisionStep;
int totalTermCountT = 0, totalTermCountC = 0, termCountT, termCountC;
int num = TestUtil.NextInt32(Random, 10, 20);
for (int i = 0; i < num; i++)
{
int lower = (int)(Random.NextDouble() * NoDocs * Distance) + StartOffset;
int upper = (int)(Random.NextDouble() * NoDocs * Distance) + StartOffset;
if (lower > upper)
{
int a = lower;
lower = upper;
upper = a;
}
BytesRef lowerBytes = new BytesRef(NumericUtils.BUF_SIZE_INT32), upperBytes = new BytesRef(NumericUtils.BUF_SIZE_INT32);
NumericUtils.Int32ToPrefixCodedBytes(lower, 0, lowerBytes);
NumericUtils.Int32ToPrefixCodedBytes(upper, 0, upperBytes);
// test inclusive range
NumericRangeQuery <int> tq = NumericRangeQuery.NewInt32Range(field, precisionStep, lower, upper, true, true);
TermRangeQuery cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, true);
TopDocs tTopDocs = Searcher.Search(tq, 1);
TopDocs cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test exclusive range
tq = NumericRangeQuery.NewInt32Range(field, precisionStep, lower, upper, false, false);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, false);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test left exclusive range
tq = NumericRangeQuery.NewInt32Range(field, precisionStep, lower, upper, false, true);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, false, true);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
// test right exclusive range
tq = NumericRangeQuery.NewInt32Range(field, precisionStep, lower, upper, true, false);
cq = new TermRangeQuery(field, lowerBytes, upperBytes, true, false);
tTopDocs = Searcher.Search(tq, 1);
cTopDocs = Searcher.Search(cq, 1);
Assert.AreEqual(cTopDocs.TotalHits, tTopDocs.TotalHits, "Returned count for NumericRangeQuery and TermRangeQuery must be equal");
totalTermCountT += termCountT = CountTerms(tq);
totalTermCountC += termCountC = CountTerms(cq);
CheckTermCounts(precisionStep, termCountT, termCountC);
}
CheckTermCounts(precisionStep, totalTermCountT, totalTermCountC);
if (VERBOSE && precisionStep != int.MaxValue)
{
Console.WriteLine("Average number of terms during random search on '" + field + "':");
Console.WriteLine(" Numeric query: " + (((double)totalTermCountT) / (num * 4)));
Console.WriteLine(" Classical query: " + (((double)totalTermCountC) / (num * 4)));
}
}
public virtual void Test()
{
var ITERS = AtLeast(1000);
var buffer = new RollingCharBuffer();
var random = Random();
for (var iter = 0; iter < ITERS; iter++)
{
var stringLen = random.NextBoolean() ? random.Next(50) : random.Next(20000);
string s;
if (stringLen == 0)
{
s = "";
}
else
{
s = TestUtil.RandomUnicodeString(random, stringLen);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter=" + iter + " s.length()=" + s.Length);
}
buffer.Reset(new StringReader(s));
var nextRead = 0;
var availCount = 0;
while (nextRead < s.Length)
{
if (VERBOSE)
{
Console.WriteLine(" cycle nextRead=" + nextRead + " avail=" + availCount);
}
if (availCount == 0 || random.NextBoolean())
{
// Read next char
if (VERBOSE)
{
Console.WriteLine(" new char");
}
assertEquals(s[nextRead], buffer.Get(nextRead));
nextRead++;
availCount++;
}
else if (random.NextBoolean())
{
// Read previous char
var pos = TestUtil.NextInt(random, nextRead - availCount, nextRead - 1);
if (VERBOSE)
{
Console.WriteLine(" old char pos=" + pos);
}
assertEquals(s[pos], buffer.Get(pos));
}
else
{
// Read slice
int length;
if (availCount == 1)
{
length = 1;
}
else
{
length = TestUtil.NextInt(random, 1, availCount);
}
int start;
if (length == availCount)
{
start = nextRead - availCount;
}
else
{
start = nextRead - availCount + random.Next(availCount - length);
}
if (VERBOSE)
{
Console.WriteLine(" slice start=" + start + " length=" + length);
}
assertEquals(s.Substring(start, length), new string(buffer.Get(start, length)));
}
if (availCount > 0 && random.Next(20) == 17)
{
var toFree = random.Next(availCount);
if (VERBOSE)
{
Console.WriteLine(" free " + toFree + " (avail=" + (availCount - toFree) + ")");
}
buffer.FreeBefore(nextRead - (availCount - toFree));
availCount -= toFree;
}
}
}
}
private void DoTestSeekDoesNotExist(Random r, int numField, IList <Term> fieldTerms, Term[] fieldTermsArray, IndexReader reader)
{
IDictionary <string, TermsEnum> tes = new Dictionary <string, TermsEnum>();
if (VERBOSE)
{
Console.WriteLine("TEST: top random seeks");
}
{
int num = AtLeast(100);
for (int iter = 0; iter < num; iter++)
{
// seek to random spot
string field = ("f" + r.Next(numField)).Intern();
Term tx = new Term(field, GetRandomString(r));
int spot = Array.BinarySearch(fieldTermsArray, tx);
if (spot < 0)
{
if (VERBOSE)
{
Console.WriteLine("TEST: non-exist seek to " + field + ":" + UnicodeUtil.ToHexString(tx.Text()));
}
// term does not exist:
TermsEnum te;
if (!tes.TryGetValue(field, out te))
{
te = MultiFields.GetTerms(reader, field).GetIterator(null);
tes[field] = te;
}
if (VERBOSE)
{
Console.WriteLine(" got enum");
}
spot = -spot - 1;
if (spot == fieldTerms.Count || !fieldTerms[spot].Field.Equals(field, StringComparison.Ordinal))
{
Assert.AreEqual(TermsEnum.SeekStatus.END, te.SeekCeil(tx.Bytes));
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, te.SeekCeil(tx.Bytes));
if (VERBOSE)
{
Console.WriteLine(" got term=" + UnicodeUtil.ToHexString(te.Term.Utf8ToString()));
Console.WriteLine(" exp term=" + UnicodeUtil.ToHexString(fieldTerms[spot].Text()));
}
Assert.AreEqual(fieldTerms[spot].Bytes, te.Term);
// now .next() this many times:
int ct = TestUtil.NextInt32(r, 5, 100);
for (int i = 0; i < ct; i++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: now next()");
}
if (1 + spot + i >= fieldTerms.Count)
{
break;
}
Term term = fieldTerms[1 + spot + i];
if (!term.Field.Equals(field, StringComparison.Ordinal))
{
Assert.IsNull(te.Next());
break;
}
else
{
BytesRef t = te.Next();
if (VERBOSE)
{
Console.WriteLine(" got term=" + (t == null ? null : UnicodeUtil.ToHexString(t.Utf8ToString())));
Console.WriteLine(" exp=" + UnicodeUtil.ToHexString(term.Text().ToString()));
}
Assert.AreEqual(term.Bytes, t);
}
}
}
}
}
}
}
public virtual void TestArbitraryFields()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int NUM_DOCS = AtLeast(27);
if (VERBOSE)
{
Console.WriteLine("TEST: " + NUM_DOCS + " docs");
}
int[] fieldsPerDoc = new int[NUM_DOCS];
int baseCount = 0;
for (int docCount = 0; docCount < NUM_DOCS; docCount++)
{
int fieldCount = TestUtil.NextInt32(Random, 1, 17);
fieldsPerDoc[docCount] = fieldCount - 1;
int finalDocCount = docCount;
if (VERBOSE)
{
Console.WriteLine("TEST: " + fieldCount + " fields in doc " + docCount);
}
int finalBaseCount = baseCount;
baseCount += fieldCount - 1;
w.AddDocument(new IterableAnonymousInnerClassHelper(this, fieldCount, finalDocCount, finalBaseCount));
}
IndexReader r = w.GetReader();
w.Dispose();
IndexSearcher s = NewSearcher(r);
int counter = 0;
for (int id = 0; id < NUM_DOCS; id++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: verify doc id=" + id + " (" + fieldsPerDoc[id] + " fields) counter=" + counter);
}
TopDocs hits = s.Search(new TermQuery(new Term("id", "" + id)), 1);
Assert.AreEqual(1, hits.TotalHits);
int docID = hits.ScoreDocs[0].Doc;
Document doc = s.Doc(docID);
int endCounter = counter + fieldsPerDoc[id];
while (counter < endCounter)
{
string name = "f" + counter;
int fieldID = counter % 10;
bool stored = (counter & 1) == 0 || fieldID == 3;
bool binary = fieldID == 3;
bool indexed = fieldID != 3;
string stringValue;
if (fieldID != 3 && fieldID != 9)
{
stringValue = "text " + counter;
}
else
{
stringValue = null;
}
// stored:
if (stored)
{
IIndexableField f = doc.GetField(name);
Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter);
if (binary)
{
Assert.IsNotNull(f, "doc " + id + " doesn't have field f" + counter);
BytesRef b = f.GetBinaryValue();
Assert.IsNotNull(b);
Assert.AreEqual(10, b.Length);
for (int idx = 0; idx < 10; idx++)
{
Assert.AreEqual((byte)(idx + counter), b.Bytes[b.Offset + idx]);
}
}
else
{
Debug.Assert(stringValue != null);
Assert.AreEqual(stringValue, f.GetStringValue());
}
}
if (indexed)
{
bool tv = counter % 2 == 1 && fieldID != 9;
if (tv)
{
Terms tfv = r.GetTermVectors(docID).GetTerms(name);
Assert.IsNotNull(tfv);
TermsEnum termsEnum = tfv.GetIterator(null);
Assert.AreEqual(new BytesRef("" + counter), termsEnum.Next());
Assert.AreEqual(1, termsEnum.TotalTermFreq);
DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, dpEnum.Freq);
Assert.AreEqual(1, dpEnum.NextPosition());
Assert.AreEqual(new BytesRef("text"), termsEnum.Next());
Assert.AreEqual(1, termsEnum.TotalTermFreq);
dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, dpEnum.Freq);
Assert.AreEqual(0, dpEnum.NextPosition());
Assert.IsNull(termsEnum.Next());
// TODO: offsets
}
else
{
Fields vectors = r.GetTermVectors(docID);
Assert.IsTrue(vectors == null || vectors.GetTerms(name) == null);
}
BooleanQuery bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST);
bq.Add(new TermQuery(new Term(name, "text")), Occur.MUST);
TopDocs hits2 = s.Search(bq, 1);
Assert.AreEqual(1, hits2.TotalHits);
Assert.AreEqual(docID, hits2.ScoreDocs[0].Doc);
bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("id", "" + id)), Occur.MUST);
bq.Add(new TermQuery(new Term(name, "" + counter)), Occur.MUST);
TopDocs hits3 = s.Search(bq, 1);
Assert.AreEqual(1, hits3.TotalHits);
Assert.AreEqual(docID, hits3.ScoreDocs[0].Doc);
}
counter++;
}
}
r.Dispose();
dir.Dispose();
}
public virtual void Test([ValueSource(typeof(ConcurrentMergeSchedulerFactories), "Values")] Func <IConcurrentMergeScheduler> newScheduler)
{
MockDirectoryWrapper dir = new MockDirectoryWrapper(Random, new MMapDirectory(CreateTempDir("4GBStoredFields")));
dir.Throttling = Throttling.NEVER;
var config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))
.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.SetRAMBufferSizeMB(256.0)
.SetMergeScheduler(newScheduler())
.SetMergePolicy(NewLogMergePolicy(false, 10))
.SetOpenMode(OpenMode.CREATE);
IndexWriter w = new IndexWriter(dir, config);
MergePolicy mp = w.Config.MergePolicy;
if (mp is LogByteSizeMergePolicy)
{
// 1 petabyte:
((LogByteSizeMergePolicy)mp).MaxMergeMB = 1024 * 1024 * 1024;
}
Document doc = new Document();
FieldType ft = new FieldType();
ft.IsIndexed = false;
ft.IsStored = true;
ft.Freeze();
int valueLength = RandomInts.RandomInt32Between(Random, 1 << 13, 1 << 20);
var value = new byte[valueLength];
for (int i = 0; i < valueLength; ++i)
{
// random so that even compressing codecs can't compress it
value[i] = (byte)Random.Next(256);
}
Field f = new Field("fld", value, ft);
doc.Add(f);
int numDocs = (int)((1L << 32) / valueLength + 100);
for (int i = 0; i < numDocs; ++i)
{
w.AddDocument(doc);
if (VERBOSE && i % (numDocs / 10) == 0)
{
Console.WriteLine(i + " of " + numDocs + "...");
}
}
w.ForceMerge(1);
w.Dispose();
if (VERBOSE)
{
bool found = false;
foreach (string file in dir.ListAll())
{
if (file.EndsWith(".fdt", StringComparison.Ordinal))
{
long fileLength = dir.FileLength(file);
if (fileLength >= 1L << 32)
{
found = true;
}
Console.WriteLine("File length of " + file + " : " + fileLength);
}
}
if (!found)
{
Console.WriteLine("No .fdt file larger than 4GB, test bug?");
}
}
DirectoryReader rd = DirectoryReader.Open(dir);
Document sd = rd.Document(numDocs - 1);
Assert.IsNotNull(sd);
Assert.AreEqual(1, sd.Fields.Count);
BytesRef valueRef = sd.GetBinaryValue("fld");
Assert.IsNotNull(valueRef);
Assert.AreEqual(new BytesRef(value), valueRef);
rd.Dispose();
dir.Dispose();
}
public virtual void _run()
{
for (int iter = 0; iter < NUM_TEST_ITER; iter++)
{
FieldData field = Fields[Random.Next(Fields.Length)];
TermsEnum termsEnum = TermsDict.GetTerms(field.FieldInfo.Name).GetIterator(null);
#pragma warning disable 612, 618
if (Si.Codec is Lucene3xCodec)
#pragma warning restore 612, 618
{
// code below expects unicode sort order
continue;
}
int upto = 0;
// Test straight enum of the terms:
while (true)
{
BytesRef term = termsEnum.Next();
if (term == null)
{
break;
}
BytesRef expected = new BytesRef(field.Terms[upto++].Text2);
Assert.IsTrue(expected.BytesEquals(term), "expected=" + expected + " vs actual " + term);
}
Assert.AreEqual(upto, field.Terms.Length);
// Test random seek:
TermData term2 = field.Terms[Random.Next(field.Terms.Length)];
TermsEnum.SeekStatus status = termsEnum.SeekCeil(new BytesRef(term2.Text2));
Assert.AreEqual(status, TermsEnum.SeekStatus.FOUND);
Assert.AreEqual(term2.Docs.Length, termsEnum.DocFreq);
if (field.OmitTF)
{
this.VerifyDocs(term2.Docs, term2.Positions, TestUtil.Docs(Random, termsEnum, null, null, DocsFlags.NONE), false);
}
else
{
this.VerifyDocs(term2.Docs, term2.Positions, termsEnum.DocsAndPositions(null, null), true);
}
// Test random seek by ord:
int idx = Random.Next(field.Terms.Length);
term2 = field.Terms[idx];
bool success = false;
try
{
termsEnum.SeekExact(idx);
success = true;
}
#pragma warning disable 168
catch (System.NotSupportedException uoe)
#pragma warning restore 168
{
// ok -- skip it
}
if (success)
{
Assert.AreEqual(status, TermsEnum.SeekStatus.FOUND);
Assert.IsTrue(termsEnum.Term.BytesEquals(new BytesRef(term2.Text2)));
Assert.AreEqual(term2.Docs.Length, termsEnum.DocFreq);
if (field.OmitTF)
{
this.VerifyDocs(term2.Docs, term2.Positions, TestUtil.Docs(Random, termsEnum, null, null, DocsFlags.NONE), false);
}
else
{
this.VerifyDocs(term2.Docs, term2.Positions, termsEnum.DocsAndPositions(null, null), true);
}
}
// Test seek to non-existent terms:
if (VERBOSE)
{
Console.WriteLine("TEST: seek non-exist terms");
}
for (int i = 0; i < 100; i++)
{
string text2 = TestUtil.RandomUnicodeString(Random) + ".";
status = termsEnum.SeekCeil(new BytesRef(text2));
Assert.IsTrue(status == TermsEnum.SeekStatus.NOT_FOUND || status == TermsEnum.SeekStatus.END);
}
// Seek to each term, backwards:
if (VERBOSE)
{
Console.WriteLine("TEST: seek terms backwards");
}
for (int i = field.Terms.Length - 1; i >= 0; i--)
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, termsEnum.SeekCeil(new BytesRef(field.Terms[i].Text2)), Thread.CurrentThread.Name + ": field=" + field.FieldInfo.Name + " term=" + field.Terms[i].Text2);
Assert.AreEqual(field.Terms[i].Docs.Length, termsEnum.DocFreq);
}
// Seek to each term by ord, backwards
for (int i = field.Terms.Length - 1; i >= 0; i--)
{
try
{
termsEnum.SeekExact(i);
Assert.AreEqual(field.Terms[i].Docs.Length, termsEnum.DocFreq);
Assert.IsTrue(termsEnum.Term.BytesEquals(new BytesRef(field.Terms[i].Text2)));
}
#pragma warning disable 168
catch (System.NotSupportedException uoe)
#pragma warning restore 168
{
}
}
// Seek to non-existent empty-string term
status = termsEnum.SeekCeil(new BytesRef(""));
Assert.IsNotNull(status);
//Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status);
// Make sure we're now pointing to first term
Assert.IsTrue(termsEnum.Term.BytesEquals(new BytesRef(field.Terms[0].Text2)));
// Test docs enum
termsEnum.SeekCeil(new BytesRef(""));
upto = 0;
do
{
term2 = field.Terms[upto];
if (Random.Next(3) == 1)
{
DocsEnum docs;
DocsEnum docsAndFreqs;
DocsAndPositionsEnum postings;
if (!field.OmitTF)
{
postings = termsEnum.DocsAndPositions(null, null);
if (postings != null)
{
docs = docsAndFreqs = postings;
}
else
{
docs = docsAndFreqs = TestUtil.Docs(Random, termsEnum, null, null, DocsFlags.FREQS);
}
}
else
{
postings = null;
docsAndFreqs = null;
docs = TestUtil.Docs(Random, termsEnum, null, null, DocsFlags.NONE);
}
Assert.IsNotNull(docs);
int upto2 = -1;
bool ended = false;
while (upto2 < term2.Docs.Length - 1)
{
// Maybe skip:
int left = term2.Docs.Length - upto2;
int doc;
if (Random.Next(3) == 1 && left >= 1)
{
int inc = 1 + Random.Next(left - 1);
upto2 += inc;
if (Random.Next(2) == 1)
{
doc = docs.Advance(term2.Docs[upto2]);
Assert.AreEqual(term2.Docs[upto2], doc);
}
else
{
doc = docs.Advance(1 + term2.Docs[upto2]);
if (doc == DocIdSetIterator.NO_MORE_DOCS)
{
// skipped past last doc
Debug.Assert(upto2 == term2.Docs.Length - 1);
ended = true;
break;
}
else
{
// skipped to next doc
Debug.Assert(upto2 < term2.Docs.Length - 1);
if (doc >= term2.Docs[1 + upto2])
{
upto2++;
}
}
}
}
else
{
doc = docs.NextDoc();
Assert.IsTrue(doc != -1);
upto2++;
}
Assert.AreEqual(term2.Docs[upto2], doc);
if (!field.OmitTF)
{
Assert.AreEqual(term2.Positions[upto2].Length, postings.Freq);
if (Random.Next(2) == 1)
{
this.VerifyPositions(term2.Positions[upto2], postings);
}
}
}
if (!ended)
{
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, docs.NextDoc());
}
}
upto++;
} while (termsEnum.Next() != null);
Assert.AreEqual(upto, field.Terms.Length);
}
}
public static void TermInfo(Store.Directory dir, Term term)
{
IndexReader reader = DirectoryReader.Open(dir);
Console.WriteLine("{0}:{1} \t totalTF = {2:#,##0} \t doc freq = {3:#,##0} \n", term.Field, term.Text(), reader.TotalTermFreq(term), reader.DocFreq(term));
}
public virtual void TestIntermediateClose()
{
Directory dir = NewDirectory();
// Test can deadlock if we use SMS:
IConcurrentMergeScheduler scheduler;
#if !FEATURE_CONCURRENTMERGESCHEDULER
scheduler = new TaskMergeScheduler();
#else
scheduler = new ConcurrentMergeScheduler();
#endif
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergeScheduler(scheduler));
writer.AddDocument(new Document());
writer.Commit();
CountdownEvent awaitEnterWarm = new CountdownEvent(1);
CountdownEvent awaitClose = new CountdownEvent(1);
AtomicBoolean triedReopen = new AtomicBoolean(false);
//TaskScheduler es = Random().NextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
TaskScheduler es = Random.NextBoolean() ? null : TaskScheduler.Default;
SearcherFactory factory = new SearcherFactoryAnonymousInnerClassHelper2(this, awaitEnterWarm, awaitClose, triedReopen, es);
SearcherManager searcherManager = Random.NextBoolean() ? new SearcherManager(dir, factory) : new SearcherManager(writer, Random.NextBoolean(), factory);
if (VERBOSE)
{
Console.WriteLine("sm created");
}
IndexSearcher searcher = searcherManager.Acquire();
try
{
assertEquals(1, searcher.IndexReader.NumDocs);
}
finally
{
searcherManager.Release(searcher);
}
writer.AddDocument(new Document());
writer.Commit();
AtomicBoolean success = new AtomicBoolean(false);
Exception[] exc = new Exception[1];
ThreadJob thread = new ThreadJob(() => new RunnableAnonymousInnerClassHelper(this, triedReopen, searcherManager, success, exc).Run());
thread.Start();
if (VERBOSE)
{
Console.WriteLine("THREAD started");
}
awaitEnterWarm.Wait();
if (VERBOSE)
{
Console.WriteLine("NOW call close");
}
searcherManager.Dispose();
awaitClose.Signal();
thread.Join();
try
{
searcherManager.Acquire();
fail("already closed");
}
#pragma warning disable 168
catch (ObjectDisposedException ex)
#pragma warning restore 168
{
// expected
}
assertFalse(success);
assertTrue(triedReopen);
assertNull("" + exc[0], exc[0]);
writer.Dispose();
dir.Dispose();
//if (es != null)
//{
// es.shutdown();
// es.awaitTermination(1, TimeUnit.SECONDS);
//}
}
private void ExecuteRandomJoin(bool multipleValuesPerDocument, int maxIndexIter, int maxSearchIter,
int numberOfDocumentsToIndex)
{
for (int indexIter = 1; indexIter <= maxIndexIter; indexIter++)
{
if (VERBOSE)
{
Console.WriteLine("indexIter=" + indexIter);
}
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(Random, dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random, MockTokenizer.KEYWORD, false))
.SetMergePolicy(NewLogMergePolicy()));
bool scoreDocsInOrder = TestJoinUtil.Random.NextBoolean();
IndexIterationContext context = CreateContext(numberOfDocumentsToIndex, w, multipleValuesPerDocument,
scoreDocsInOrder);
IndexReader topLevelReader = w.GetReader();
w.Dispose();
for (int searchIter = 1; searchIter <= maxSearchIter; searchIter++)
{
if (VERBOSE)
{
Console.WriteLine("searchIter=" + searchIter);
}
IndexSearcher indexSearcher = NewSearcher(topLevelReader);
int r = Random.Next(context.RandomUniqueValues.Length);
bool from = context.RandomFrom[r];
string randomValue = context.RandomUniqueValues[r];
FixedBitSet expectedResult = CreateExpectedResult(randomValue, from, indexSearcher.IndexReader,
context);
Query actualQuery = new TermQuery(new Term("value", randomValue));
if (VERBOSE)
{
Console.WriteLine("actualQuery=" + actualQuery);
}
var scoreModeLength = Enum.GetNames(typeof(ScoreMode)).Length;
ScoreMode scoreMode = (ScoreMode)Random.Next(scoreModeLength);
if (VERBOSE)
{
Console.WriteLine("scoreMode=" + scoreMode);
}
Query joinQuery;
if (from)
{
joinQuery = JoinUtil.CreateJoinQuery("from", multipleValuesPerDocument, "to", actualQuery,
indexSearcher, scoreMode);
}
else
{
joinQuery = JoinUtil.CreateJoinQuery("to", multipleValuesPerDocument, "from", actualQuery,
indexSearcher, scoreMode);
}
if (VERBOSE)
{
Console.WriteLine("joinQuery=" + joinQuery);
}
// Need to know all documents that have matches. TopDocs doesn't give me that and then I'd be also testing TopDocsCollector...
FixedBitSet actualResult = new FixedBitSet(indexSearcher.IndexReader.MaxDoc);
TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.Create(10, false);
indexSearcher.Search(joinQuery,
new CollectorAnonymousInnerClassHelper2(this, scoreDocsInOrder, context, actualResult,
topScoreDocCollector));
// Asserting bit set...
if (VERBOSE)
{
Console.WriteLine("expected cardinality:" + expectedResult.Cardinality());
DocIdSetIterator iterator = expectedResult.GetIterator();
for (int doc = iterator.NextDoc();
doc != DocIdSetIterator.NO_MORE_DOCS;
doc = iterator.NextDoc())
{
Console.WriteLine(string.Format("Expected doc[{0}] with id value {1}", doc, indexSearcher.Doc(doc).Get("id")));
}
Console.WriteLine("actual cardinality:" + actualResult.Cardinality());
iterator = actualResult.GetIterator();
for (int doc = iterator.NextDoc();
doc != DocIdSetIterator.NO_MORE_DOCS;
doc = iterator.NextDoc())
{
Console.WriteLine(string.Format("Actual doc[{0}] with id value {1}", doc, indexSearcher.Doc(doc).Get("id")));
}
}
assertEquals(expectedResult, actualResult);
// Asserting TopDocs...
TopDocs expectedTopDocs = CreateExpectedTopDocs(randomValue, from, scoreMode, context);
TopDocs actualTopDocs = topScoreDocCollector.GetTopDocs();
assertEquals(expectedTopDocs.TotalHits, actualTopDocs.TotalHits);
assertEquals(expectedTopDocs.ScoreDocs.Length, actualTopDocs.ScoreDocs.Length);
if (scoreMode == ScoreMode.None)
{
continue;
}
assertEquals(expectedTopDocs.MaxScore, actualTopDocs.MaxScore, 0.0f);
for (int i = 0; i < expectedTopDocs.ScoreDocs.Length; i++)
{
if (VERBOSE)
{
string.Format("Expected doc: {0} | Actual doc: {1}\n", expectedTopDocs.ScoreDocs[i].Doc, actualTopDocs.ScoreDocs[i].Doc);
string.Format("Expected score: {0} | Actual score: {1}\n", expectedTopDocs.ScoreDocs[i].Score, actualTopDocs.ScoreDocs[i].Score);
}
assertEquals(expectedTopDocs.ScoreDocs[i].Doc, actualTopDocs.ScoreDocs[i].Doc);
assertEquals(expectedTopDocs.ScoreDocs[i].Score, actualTopDocs.ScoreDocs[i].Score, 0.0f);
Explanation explanation = indexSearcher.Explain(joinQuery, expectedTopDocs.ScoreDocs[i].Doc);
assertEquals(expectedTopDocs.ScoreDocs[i].Score, explanation.Value, 0.0f);
}
}
topLevelReader.Dispose();
dir.Dispose();
}
}
public virtual void TestCopyBytesMem()
{
int num = AtLeast(10);
for (int iter = 0; iter < num; iter++)
{
Directory dir = NewDirectory();
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + " dir=" + dir);
}
// make random file
IndexOutput @out = dir.CreateOutput("test", NewIOContext(Random));
var bytes = new byte[TestUtil.NextInt32(Random, 1, 77777)];
int size = TestUtil.NextInt32(Random, 1, 1777777);
int upto = 0;
int byteUpto = 0;
while (upto < size)
{
bytes[byteUpto++] = Value(upto);
upto++;
if (byteUpto == bytes.Length)
{
@out.WriteBytes(bytes, 0, bytes.Length);
byteUpto = 0;
}
}
@out.WriteBytes(bytes, 0, byteUpto);
Assert.AreEqual(size, @out.GetFilePointer());
@out.Dispose();
Assert.AreEqual(size, dir.FileLength("test"));
// copy from test -> test2
IndexInput @in = dir.OpenInput("test", NewIOContext(Random));
@out = dir.CreateOutput("test2", NewIOContext(Random));
upto = 0;
while (upto < size)
{
if (Random.NextBoolean())
{
@out.WriteByte(@in.ReadByte());
upto++;
}
else
{
int chunk = Math.Min(TestUtil.NextInt32(Random, 1, bytes.Length), size - upto);
@out.CopyBytes(@in, chunk);
upto += chunk;
}
}
Assert.AreEqual(size, upto);
@out.Dispose();
@in.Dispose();
// verify
IndexInput in2 = dir.OpenInput("test2", NewIOContext(Random));
upto = 0;
while (upto < size)
{
if (Random.NextBoolean())
{
var v = in2.ReadByte();
Assert.AreEqual(Value(upto), v);
upto++;
}
else
{
int limit = Math.Min(TestUtil.NextInt32(Random, 1, bytes.Length), size - upto);
in2.ReadBytes(bytes, 0, limit);
for (int byteIdx = 0; byteIdx < limit; byteIdx++)
{
Assert.AreEqual(Value(upto), bytes[byteIdx]);
upto++;
}
}
}
in2.Dispose();
dir.DeleteFile("test");
dir.DeleteFile("test2");
dir.Dispose();
}
}
