private void TestRandomSeeks(IndexReader r, params string[] validTermStrings)
{
BytesRef[] validTerms = new BytesRef[validTermStrings.Length];
for (int termIDX = 0; termIDX < validTermStrings.Length; termIDX++)
{
validTerms[termIDX] = new BytesRef(validTermStrings[termIDX]);
}
Array.Sort(validTerms);
if (VERBOSE)
{
Console.WriteLine("TEST: " + validTerms.Length + " terms:");
foreach (BytesRef t in validTerms)
{
Console.WriteLine(" " + t.Utf8ToString() + " " + t);
}
}
TermsEnum te = MultiFields.GetTerms(r, FIELD).GetIterator(null);
int END_LOC = -validTerms.Length - 1;
IList <TermAndState> termStates = new List <TermAndState>();
for (int iter = 0; iter < 100 * RANDOM_MULTIPLIER; iter++)
{
BytesRef t;
int loc;
TermState termState;
if (Random.Next(6) == 4)
{
// pick term that doens't exist:
t = GetNonExistTerm(validTerms);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: invalid term=" + t.Utf8ToString());
}
loc = Array.BinarySearch(validTerms, t);
}
else if (termStates.Count != 0 && Random.Next(4) == 1)
{
TermAndState ts = termStates[Random.Next(termStates.Count)];
t = ts.Term;
loc = Array.BinarySearch(validTerms, t);
Assert.IsTrue(loc >= 0);
termState = ts.State;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid termState term=" + t.Utf8ToString());
}
}
else
{
// pick valid term
loc = Random.Next(validTerms.Length);
t = BytesRef.DeepCopyOf(validTerms[loc]);
termState = null;
if (VERBOSE)
{
Console.WriteLine("\nTEST: valid term=" + t.Utf8ToString());
}
}
// seekCeil or seekExact:
bool doSeekExact = Random.NextBoolean();
if (termState != null)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact termState");
}
te.SeekExact(t, termState);
}
else if (doSeekExact)
{
if (VERBOSE)
{
Console.WriteLine(" seekExact");
}
Assert.AreEqual(loc >= 0, te.SeekExact(t));
}
else
{
if (VERBOSE)
{
Console.WriteLine(" seekCeil");
}
TermsEnum.SeekStatus result = te.SeekCeil(t);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (loc >= 0)
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, result);
}
else if (loc == END_LOC)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, result);
}
else
{
Debug.Assert(loc >= -validTerms.Length);
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, result);
}
}
if (loc >= 0)
{
Assert.AreEqual(t, te.Term);
}
else if (doSeekExact)
{
// TermsEnum is unpositioned if seekExact returns false
continue;
}
else if (loc == END_LOC)
{
continue;
}
else
{
loc = -loc - 1;
Assert.AreEqual(validTerms[loc], te.Term);
}
// Do a bunch of next's after the seek
int numNext = Random.Next(validTerms.Length);
for (int nextCount = 0; nextCount < numNext; nextCount++)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: next loc=" + loc + " of " + validTerms.Length);
}
BytesRef t2 = te.Next();
loc++;
if (loc == validTerms.Length)
{
Assert.IsNull(t2);
break;
}
else
{
Assert.AreEqual(validTerms[loc], t2);
if (Random.Next(40) == 17 && termStates.Count < 100)
{
termStates.Add(new TermAndState(validTerms[loc], te.GetTermState()));
}
}
}
}
}
public virtual void TestIntersectRandom()
{
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
int numTerms = AtLeast(300);
//final int numTerms = 50;
ISet <string> terms = new JCG.HashSet <string>();
ICollection <string> pendingTerms = new List <string>();
IDictionary <BytesRef, int?> termToID = new Dictionary <BytesRef, int?>();
int id = 0;
while (terms.Count != numTerms)
{
string s = RandomString;
if (!terms.Contains(s))
{
terms.Add(s);
pendingTerms.Add(s);
if (Random.Next(20) == 7)
{
AddDoc(w, pendingTerms, termToID, id++);
}
}
}
AddDoc(w, pendingTerms, termToID, id++);
BytesRef[] termsArray = new BytesRef[terms.Count];
ISet <BytesRef> termsSet = new JCG.HashSet <BytesRef>();
{
int upto = 0;
foreach (string s in terms)
{
BytesRef b = new BytesRef(s);
termsArray[upto++] = b;
termsSet.Add(b);
}
Array.Sort(termsArray);
}
if (VERBOSE)
{
Console.WriteLine("\nTEST: indexed terms (unicode order):");
foreach (BytesRef t in termsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + " -> id:" + termToID[t]);
}
}
IndexReader r = w.GetReader();
w.Dispose();
// NOTE: intentional insanity!!
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
for (int iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
{
// TODO: can we also test infinite As here...?
// From the random terms, pick some ratio and compile an
// automaton:
ISet <string> acceptTerms = new JCG.HashSet <string>();
JCG.SortedSet <BytesRef> sortedAcceptTerms = new JCG.SortedSet <BytesRef>();
double keepPct = Random.NextDouble();
Automaton a;
if (iter == 0)
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: empty automaton");
}
a = BasicAutomata.MakeEmpty();
}
else
{
if (VERBOSE)
{
Console.WriteLine("\nTEST: keepPct=" + keepPct);
}
foreach (string s in terms)
{
string s2;
if (Random.NextDouble() <= keepPct)
{
s2 = s;
}
else
{
s2 = RandomString;
}
acceptTerms.Add(s2);
sortedAcceptTerms.Add(new BytesRef(s2));
}
a = BasicAutomata.MakeStringUnion(sortedAcceptTerms);
}
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: reduce the automaton");
}
a.Reduce();
}
CompiledAutomaton c = new CompiledAutomaton(a, true, false);
BytesRef[] acceptTermsArray = new BytesRef[acceptTerms.Count];
ISet <BytesRef> acceptTermsSet = new JCG.HashSet <BytesRef>();
int upto = 0;
foreach (string s in acceptTerms)
{
BytesRef b = new BytesRef(s);
acceptTermsArray[upto++] = b;
acceptTermsSet.Add(b);
Assert.IsTrue(Accepts(c, b));
}
Array.Sort(acceptTermsArray);
if (VERBOSE)
{
Console.WriteLine("\nTEST: accept terms (unicode order):");
foreach (BytesRef t in acceptTermsArray)
{
Console.WriteLine(" " + t.Utf8ToString() + (termsSet.Contains(t) ? " (exists)" : ""));
}
Console.WriteLine(a.ToDot());
}
for (int iter2 = 0; iter2 < 100; iter2++)
{
BytesRef startTerm = acceptTermsArray.Length == 0 || Random.NextBoolean() ? null : acceptTermsArray[Random.Next(acceptTermsArray.Length)];
if (VERBOSE)
{
Console.WriteLine("\nTEST: iter2=" + iter2 + " startTerm=" + (startTerm == null ? "<null>" : startTerm.Utf8ToString()));
if (startTerm != null)
{
int state = c.RunAutomaton.InitialState;
for (int idx = 0; idx < startTerm.Length; idx++)
{
int label = startTerm.Bytes[startTerm.Offset + idx] & 0xff;
Console.WriteLine(" state=" + state + " label=" + label);
state = c.RunAutomaton.Step(state, label);
Assert.IsTrue(state != -1);
}
Console.WriteLine(" state=" + state);
}
}
TermsEnum te = MultiFields.GetTerms(r, "f").Intersect(c, startTerm);
int loc;
if (startTerm == null)
{
loc = 0;
}
else
{
loc = Array.BinarySearch(termsArray, BytesRef.DeepCopyOf(startTerm));
if (loc < 0)
{
loc = -(loc + 1);
}
else
{
// startTerm exists in index
loc++;
}
}
while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]))
{
loc++;
}
DocsEnum docsEnum = null;
while (loc < termsArray.Length)
{
BytesRef expected = termsArray[loc];
BytesRef actual = te.Next();
if (VERBOSE)
{
Console.WriteLine("TEST: next() expected=" + expected.Utf8ToString() + " actual=" + (actual == null ? "null" : actual.Utf8ToString()));
}
Assert.AreEqual(expected, actual);
Assert.AreEqual(1, te.DocFreq);
docsEnum = TestUtil.Docs(Random, te, null, docsEnum, DocsFlags.NONE);
int docID = docsEnum.NextDoc();
Assert.IsTrue(docID != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(docIDToID.Get(docID), (int)termToID[expected]);
do
{
loc++;
} while (loc < termsArray.Length && !acceptTermsSet.Contains(termsArray[loc]));
}
Assert.IsNull(te.Next());
}
}
r.Dispose();
dir.Dispose();
}
public virtual void Test()
{
Random random = new Random(Random.Next());
LineFileDocs docs = new LineFileDocs(random, DefaultCodecSupportsDocValues);
Directory d = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(LuceneTestCase.Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(LuceneTestCase.Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
LuceneTestCase.Random, d, analyzer);
int numDocs = AtLeast(10);
for (int docCount = 0; docCount < numDocs; docCount++)
{
w.AddDocument(docs.NextDoc());
}
IndexReader r = w.GetReader();
w.Dispose();
List <BytesRef> terms = new List <BytesRef>();
TermsEnum termsEnum = MultiFields.GetTerms(r, "body").GetIterator(null);
BytesRef term;
while ((term = termsEnum.Next()) != null)
{
terms.Add(BytesRef.DeepCopyOf(term));
}
if (VERBOSE)
{
Console.WriteLine("TEST: " + terms.Count + " terms");
}
int upto = -1;
int iters = AtLeast(200);
for (int iter = 0; iter < iters; iter++)
{
bool isEnd;
if (upto != -1 && LuceneTestCase.Random.NextBoolean())
{
// next
if (VERBOSE)
{
Console.WriteLine("TEST: iter next");
}
isEnd = termsEnum.Next() == null;
upto++;
if (isEnd)
{
if (VERBOSE)
{
Console.WriteLine(" end");
}
Assert.AreEqual(upto, terms.Count);
upto = -1;
}
else
{
if (VERBOSE)
{
Console.WriteLine(" got term=" + termsEnum.Term.Utf8ToString() + " expected=" + terms[upto].Utf8ToString());
}
Assert.IsTrue(upto < terms.Count);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
BytesRef target;
string exists;
if (LuceneTestCase.Random.NextBoolean())
{
// likely fake term
if (LuceneTestCase.Random.NextBoolean())
{
target = new BytesRef(TestUtil.RandomSimpleString(LuceneTestCase.Random));
}
else
{
target = new BytesRef(TestUtil.RandomRealisticUnicodeString(LuceneTestCase.Random));
}
exists = "likely not";
}
else
{
// real term
target = terms[LuceneTestCase.Random.Next(terms.Count)];
exists = "yes";
}
upto = terms.BinarySearch(target);
if (LuceneTestCase.Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekCeil target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekCeil
TermsEnum.SeekStatus status = termsEnum.SeekCeil(target);
if (VERBOSE)
{
Console.WriteLine(" got " + status);
}
if (upto < 0)
{
upto = -(upto + 1);
if (upto >= terms.Count)
{
Assert.AreEqual(TermsEnum.SeekStatus.END, status);
upto = -1;
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.NOT_FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
Assert.AreEqual(TermsEnum.SeekStatus.FOUND, status);
Assert.AreEqual(terms[upto], termsEnum.Term);
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter seekExact target=" + target.Utf8ToString() + " exists=" + exists);
}
// seekExact
bool result = termsEnum.SeekExact(target);
if (VERBOSE)
{
Console.WriteLine(" got " + result);
}
if (upto < 0)
{
Assert.IsFalse(result);
upto = -1;
}
else
{
Assert.IsTrue(result);
Assert.AreEqual(target, termsEnum.Term);
}
}
}
}
r.Dispose();
d.Dispose();
docs.Dispose();
}
private void Verify(AtomicReader r, int[][] idToOrds, BytesRef[] termsArray, BytesRef prefixRef)
{
DocTermOrds dto = new DocTermOrds(r, r.LiveDocs, "field", prefixRef, int.MaxValue, TestUtil.NextInt32(Random, 2, 10));
FieldCache.Int32s docIDToID = FieldCache.DEFAULT.GetInt32s(r, "id", false);
/*
* for(int docID=0;docID<subR.MaxDoc;docID++) {
* System.out.println(" docID=" + docID + " id=" + docIDToID[docID]);
* }
*/
if (Verbose)
{
Console.WriteLine("TEST: verify prefix=" + (prefixRef == null ? "null" : prefixRef.Utf8ToString()));
Console.WriteLine("TEST: all TERMS:");
TermsEnum allTE = MultiFields.GetTerms(r, "field").GetIterator(null);
int ord = 0;
while (allTE.Next() != null)
{
Console.WriteLine(" ord=" + (ord++) + " term=" + allTE.Term.Utf8ToString());
}
}
//final TermsEnum te = subR.Fields.Terms("field").iterator();
TermsEnum te = dto.GetOrdTermsEnum(r);
if (dto.NumTerms == 0)
{
if (prefixRef == null)
{
Assert.IsNull(MultiFields.GetTerms(r, "field"));
}
else
{
Terms terms = MultiFields.GetTerms(r, "field");
if (terms != null)
{
TermsEnum termsEnum = terms.GetIterator(null);
TermsEnum.SeekStatus result = termsEnum.SeekCeil(prefixRef);
if (result != TermsEnum.SeekStatus.END)
{
Assert.IsFalse(StringHelper.StartsWith(termsEnum.Term, prefixRef), "term=" + termsEnum.Term.Utf8ToString() + " matches prefix=" + prefixRef.Utf8ToString());
}
else
{
// ok
}
}
else
{
// ok
}
}
return;
}
if (Verbose)
{
Console.WriteLine("TEST: TERMS:");
te.SeekExact(0);
while (true)
{
Console.WriteLine(" ord=" + te.Ord + " term=" + te.Term.Utf8ToString());
if (te.Next() == null)
{
break;
}
}
}
SortedSetDocValues iter = dto.GetIterator(r);
for (int docID = 0; docID < r.MaxDoc; docID++)
{
if (Verbose)
{
Console.WriteLine("TEST: docID=" + docID + " of " + r.MaxDoc + " (id=" + docIDToID.Get(docID) + ")");
}
iter.SetDocument(docID);
int[] answers = idToOrds[docIDToID.Get(docID)];
int upto = 0;
long ord;
while ((ord = iter.NextOrd()) != SortedSetDocValues.NO_MORE_ORDS)
{
te.SeekExact(ord);
BytesRef expected = termsArray[answers[upto++]];
if (Verbose)
{
Console.WriteLine(" exp=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString());
}
Assert.AreEqual(expected, te.Term, "expected=" + expected.Utf8ToString() + " actual=" + te.Term.Utf8ToString() + " ord=" + ord);
}
Assert.AreEqual(answers.Length, upto);
}
}