public static void PurgeFieldCache(IndexReader r)
{
// this is just a hack, to get an atomic reader that contains all subreaders for insanity checks
FieldCache.DEFAULT.PurgeByCacheKey(SlowCompositeReaderWrapper.Wrap(r).CoreCacheKey);
}
private IndexIterationContext CreateContext(int nDocs, RandomIndexWriter fromWriter, RandomIndexWriter toWriter,
bool multipleValuesPerDocument, bool scoreDocsInOrder)
{
IndexIterationContext context = new IndexIterationContext();
int numRandomValues = nDocs / 2;
context.RandomUniqueValues = new string[numRandomValues];
ISet <string> trackSet = new HashSet <string>();
context.RandomFrom = new bool[numRandomValues];
for (int i = 0; i < numRandomValues; i++)
{
string uniqueRandomValue;
do
{
uniqueRandomValue = TestUtil.RandomRealisticUnicodeString(Random());
// uniqueRandomValue = TestUtil.randomSimpleString(random);
} while ("".Equals(uniqueRandomValue, StringComparison.Ordinal) || trackSet.Contains(uniqueRandomValue));
// Generate unique values and empty strings aren't allowed.
trackSet.Add(uniqueRandomValue);
context.RandomFrom[i] = Random().NextBoolean();
context.RandomUniqueValues[i] = uniqueRandomValue;
}
RandomDoc[] docs = new RandomDoc[nDocs];
for (int i = 0; i < nDocs; i++)
{
string id = Convert.ToString(i);
int randomI = Random().Next(context.RandomUniqueValues.Length);
string value = context.RandomUniqueValues[randomI];
Document document = new Document();
document.Add(NewTextField(Random(), "id", id, Field.Store.NO));
document.Add(NewTextField(Random(), "value", value, Field.Store.NO));
bool from = context.RandomFrom[randomI];
int numberOfLinkValues = multipleValuesPerDocument ? 2 + Random().Next(10) : 1;
docs[i] = new RandomDoc(id, numberOfLinkValues, value, from);
for (int j = 0; j < numberOfLinkValues; j++)
{
string linkValue = context.RandomUniqueValues[Random().Next(context.RandomUniqueValues.Length)];
docs[i].LinkValues.Add(linkValue);
if (from)
{
if (!context.FromDocuments.ContainsKey(linkValue))
{
context.FromDocuments[linkValue] = new List <RandomDoc>();
}
if (!context.RandomValueFromDocs.ContainsKey(value))
{
context.RandomValueFromDocs[value] = new List <RandomDoc>();
}
context.FromDocuments[linkValue].Add(docs[i]);
context.RandomValueFromDocs[value].Add(docs[i]);
document.Add(NewTextField(Random(), "from", linkValue, Field.Store.NO));
}
else
{
if (!context.ToDocuments.ContainsKey(linkValue))
{
context.ToDocuments[linkValue] = new List <RandomDoc>();
}
if (!context.RandomValueToDocs.ContainsKey(value))
{
context.RandomValueToDocs[value] = new List <RandomDoc>();
}
context.ToDocuments[linkValue].Add(docs[i]);
context.RandomValueToDocs[value].Add(docs[i]);
document.Add(NewTextField(Random(), "to", linkValue, Field.Store.NO));
}
}
RandomIndexWriter w;
if (from)
{
w = fromWriter;
}
else
{
w = toWriter;
}
w.AddDocument(document);
if (Random().Next(10) == 4)
{
w.Commit();
}
if (VERBOSE)
{
Console.WriteLine("Added document[" + docs[i].Id + "]: " + document);
}
}
// Pre-compute all possible hits for all unique random values. On top of this also compute all possible score for
// any ScoreMode.
IndexSearcher fromSearcher = NewSearcher(fromWriter.Reader);
IndexSearcher toSearcher = NewSearcher(toWriter.Reader);
for (int i = 0; i < context.RandomUniqueValues.Length; i++)
{
string uniqueRandomValue = context.RandomUniqueValues[i];
string fromField;
string toField;
IDictionary <string, IDictionary <int, JoinScore> > queryVals;
if (context.RandomFrom[i])
{
fromField = "from";
toField = "to";
queryVals = context.FromHitsToJoinScore;
}
else
{
fromField = "to";
toField = "from";
queryVals = context.ToHitsToJoinScore;
}
IDictionary <BytesRef, JoinScore> joinValueToJoinScores = new Dictionary <BytesRef, JoinScore>();
if (multipleValuesPerDocument)
{
fromSearcher.Search(new TermQuery(new Term("value", uniqueRandomValue)),
new CollectorAnonymousInnerClassHelper3(this, context, fromField, joinValueToJoinScores));
}
else
{
fromSearcher.Search(new TermQuery(new Term("value", uniqueRandomValue)),
new CollectorAnonymousInnerClassHelper4(this, context, fromField, joinValueToJoinScores));
}
IDictionary <int, JoinScore> docToJoinScore = new Dictionary <int, JoinScore>();
if (multipleValuesPerDocument)
{
if (scoreDocsInOrder)
{
AtomicReader slowCompositeReader = SlowCompositeReaderWrapper.Wrap(toSearcher.IndexReader);
Terms terms = slowCompositeReader.GetTerms(toField);
if (terms != null)
{
DocsEnum docsEnum = null;
TermsEnum termsEnum = null;
SortedSet <BytesRef> joinValues =
new SortedSet <BytesRef>(BytesRef.UTF8SortedAsUnicodeComparer);
joinValues.AddAll(joinValueToJoinScores.Keys);
foreach (BytesRef joinValue in joinValues)
{
termsEnum = terms.GetIterator(termsEnum);
if (termsEnum.SeekExact(joinValue))
{
docsEnum = termsEnum.Docs(slowCompositeReader.LiveDocs, docsEnum, DocsFlags.NONE);
JoinScore joinScore = joinValueToJoinScores[joinValue];
for (int doc = docsEnum.NextDoc();
doc != DocIdSetIterator.NO_MORE_DOCS;
doc = docsEnum.NextDoc())
{
// First encountered join value determines the score.
// Something to keep in mind for many-to-many relations.
if (!docToJoinScore.ContainsKey(doc))
{
docToJoinScore[doc] = joinScore;
}
}
}
}
}
}
else
{
toSearcher.Search(new MatchAllDocsQuery(),
new CollectorAnonymousInnerClassHelper5(this, context, toField, joinValueToJoinScores,
docToJoinScore));
}
}
else
{
toSearcher.Search(new MatchAllDocsQuery(),
new CollectorAnonymousInnerClassHelper6(this, toField, joinValueToJoinScores,
docToJoinScore));
}
queryVals[uniqueRandomValue] = docToJoinScore;
}
fromSearcher.IndexReader.Dispose();
toSearcher.IndexReader.Dispose();
return(context);
}
public virtual void TestPayloadsPos0()
{
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random, dir, new MockPayloadAnalyzer(), Similarity, TimeZone);
Document doc = new Document();
doc.Add(new TextField("content", new StringReader("a a b c d e a f g h i j a b k k")));
writer.AddDocument(doc);
IndexReader readerFromWriter = writer.GetReader();
AtomicReader r = SlowCompositeReaderWrapper.Wrap(readerFromWriter);
DocsAndPositionsEnum tp = r.GetTermPositionsEnum(new Term("content", "a"));
int count = 0;
Assert.IsTrue(tp.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
// "a" occurs 4 times
Assert.AreEqual(4, tp.Freq);
Assert.AreEqual(0, tp.NextPosition());
Assert.AreEqual(1, tp.NextPosition());
Assert.AreEqual(3, tp.NextPosition());
Assert.AreEqual(6, tp.NextPosition());
// only one doc has "a"
Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, tp.NextDoc());
IndexSearcher @is = NewSearcher(readerFromWriter);
SpanTermQuery stq1 = new SpanTermQuery(new Term("content", "a"));
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = new SpanQuery[] { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 30, false);
count = 0;
bool sawZero = false;
if (VERBOSE)
{
Console.WriteLine("\ngetPayloadSpans test");
}
Search.Spans.Spans pspans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq);
while (pspans.Next())
{
if (VERBOSE)
{
Console.WriteLine("doc " + pspans.Doc + ": span " + pspans.Start + " to " + pspans.End);
}
var payloads = pspans.GetPayload();
sawZero |= pspans.Start == 0;
foreach (var bytes in payloads)
{
count++;
if (VERBOSE)
{
Console.WriteLine(" payload: " + Encoding.UTF8.GetString(bytes));
}
}
}
Assert.IsTrue(sawZero);
Assert.AreEqual(5, count);
// System.out.println("\ngetSpans test");
Search.Spans.Spans spans = MultiSpansWrapper.Wrap(@is.TopReaderContext, snq);
count = 0;
sawZero = false;
while (spans.Next())
{
count++;
sawZero |= spans.Start == 0;
// System.out.println(spans.Doc() + " - " + spans.Start() + " - " +
// spans.End());
}
Assert.AreEqual(4, count);
Assert.IsTrue(sawZero);
// System.out.println("\nPayloadSpanUtil test");
sawZero = false;
PayloadSpanUtil psu = new PayloadSpanUtil(@is.TopReaderContext);
var pls = psu.GetPayloadsForQuery(snq);
count = pls.Count;
foreach (var bytes in pls)
{
string s = Encoding.UTF8.GetString(bytes);
//System.out.println(s);
sawZero |= s.Equals("pos: 0", StringComparison.Ordinal);
}
Assert.AreEqual(5, count);
Assert.IsTrue(sawZero);
writer.Dispose();
@is.IndexReader.Dispose();
dir.Dispose();
}
private void DuellReaders(CompositeReader other, AtomicReader memIndexReader)
{
AtomicReader competitor = SlowCompositeReaderWrapper.Wrap(other);
Fields memFields = memIndexReader.Fields;
foreach (string field in competitor.Fields)
{
Terms memTerms = memFields.GetTerms(field);
Terms iwTerms = memIndexReader.GetTerms(field);
if (iwTerms == null)
{
assertNull(memTerms);
}
else
{
NumericDocValues normValues = competitor.GetNormValues(field);
NumericDocValues memNormValues = memIndexReader.GetNormValues(field);
if (normValues != null)
{
// mem idx always computes norms on the fly
assertNotNull(memNormValues);
assertEquals(normValues.Get(0), memNormValues.Get(0));
}
assertNotNull(memTerms);
assertEquals(iwTerms.DocCount, memTerms.DocCount);
assertEquals(iwTerms.SumDocFreq, memTerms.SumDocFreq);
assertEquals(iwTerms.SumTotalTermFreq, memTerms.SumTotalTermFreq);
TermsEnum iwTermsIter = iwTerms.GetIterator(null);
TermsEnum memTermsIter = memTerms.GetIterator(null);
if (iwTerms.HasPositions)
{
bool offsets = iwTerms.HasOffsets && memTerms.HasOffsets;
while (iwTermsIter.Next() != null)
{
assertNotNull(memTermsIter.Next());
assertEquals(iwTermsIter.Term, memTermsIter.Term);
DocsAndPositionsEnum iwDocsAndPos = iwTermsIter.DocsAndPositions(null, null);
DocsAndPositionsEnum memDocsAndPos = memTermsIter.DocsAndPositions(null, null);
while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS)
{
assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc());
assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq);
for (int i = 0; i < iwDocsAndPos.Freq; i++)
{
assertEquals("term: " + iwTermsIter.Term.Utf8ToString(), iwDocsAndPos.NextPosition(), memDocsAndPos.NextPosition());
if (offsets)
{
assertEquals(iwDocsAndPos.StartOffset, memDocsAndPos.StartOffset);
assertEquals(iwDocsAndPos.EndOffset, memDocsAndPos.EndOffset);
}
}
}
}
}
else
{
while (iwTermsIter.Next() != null)
{
assertEquals(iwTermsIter.Term, memTermsIter.Term);
DocsEnum iwDocsAndPos = iwTermsIter.Docs(null, null);
DocsEnum memDocsAndPos = memTermsIter.Docs(null, null);
while (iwDocsAndPos.NextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS)
{
assertEquals(iwDocsAndPos.DocID, memDocsAndPos.NextDoc());
assertEquals(iwDocsAndPos.Freq, memDocsAndPos.Freq);
}
}
}
}
}
}
public override void BeforeClass()
{
base.BeforeClass();
NUM_DOCS = AtLeast(500);
NUM_ORDS = AtLeast(2);
Directory = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random, Directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy()));
long theLong = long.MaxValue;
double theDouble = double.MaxValue;
sbyte theByte = sbyte.MaxValue;
short theShort = short.MaxValue;
int theInt = int.MaxValue;
float theFloat = float.MaxValue;
UnicodeStrings = new string[NUM_DOCS];
//MultiValued = new BytesRef[NUM_DOCS, NUM_ORDS];
MultiValued = RectangularArrays.ReturnRectangularArray <BytesRef>(NUM_DOCS, NUM_ORDS);
if (VERBOSE)
{
Console.WriteLine("TEST: setUp");
}
for (int i = 0; i < NUM_DOCS; i++)
{
Document doc = new Document();
doc.Add(NewStringField("theLong", (theLong--).ToString(CultureInfo.InvariantCulture), Field.Store.NO));
doc.Add(NewStringField("theDouble", (theDouble--).ToString("R", CultureInfo.InvariantCulture), Field.Store.NO));
doc.Add(NewStringField("theByte", (theByte--).ToString(CultureInfo.InvariantCulture), Field.Store.NO));
doc.Add(NewStringField("theShort", (theShort--).ToString(CultureInfo.InvariantCulture), Field.Store.NO));
doc.Add(NewStringField("theInt", (theInt--).ToString(CultureInfo.InvariantCulture), Field.Store.NO));
doc.Add(NewStringField("theFloat", (theFloat--).ToString("R", CultureInfo.InvariantCulture), Field.Store.NO));
if (i % 2 == 0)
{
doc.Add(NewStringField("sparse", (i).ToString(CultureInfo.InvariantCulture), Field.Store.NO));
}
if (i % 2 == 0)
{
doc.Add(new Int32Field("numInt", i, Field.Store.NO));
}
// sometimes skip the field:
if (Random.Next(40) != 17)
{
UnicodeStrings[i] = GenerateString(i);
doc.Add(NewStringField("theRandomUnicodeString", UnicodeStrings[i], Field.Store.YES));
}
// sometimes skip the field:
if (Random.Next(10) != 8)
{
for (int j = 0; j < NUM_ORDS; j++)
{
string newValue = GenerateString(i);
MultiValued[i][j] = new BytesRef(newValue);
doc.Add(NewStringField("theRandomUnicodeMultiValuedField", newValue, Field.Store.YES));
}
Array.Sort(MultiValued[i]);
}
writer.AddDocument(doc);
}
IndexReader r = writer.GetReader();
Reader = SlowCompositeReaderWrapper.Wrap(r);
writer.Dispose();
}
public void TestRandomIndex()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir, analyzer);
CreateRandomIndex(AtLeast(50), w, Random.NextInt64());
DirectoryReader reader = w.GetReader();
AtomicReader wrapper = SlowCompositeReaderWrapper.Wrap(reader);
string field = @"body";
Terms terms = wrapper.GetTerms(field);
var lowFreqQueue = new PriorityQueueAnonymousClass(5);
var highFreqQueue = new PriorityQueueAnonymousClass1(5);
try
{
TermsEnum iterator = terms.GetEnumerator();
while (iterator.MoveNext())
{
if (highFreqQueue.Count < 5)
{
highFreqQueue.Add(new TermAndFreq(
BytesRef.DeepCopyOf(iterator.Term), iterator.DocFreq));
lowFreqQueue.Add(new TermAndFreq(
BytesRef.DeepCopyOf(iterator.Term), iterator.DocFreq));
}
else
{
if (highFreqQueue.Top.freq < iterator.DocFreq)
{
highFreqQueue.Top.freq = iterator.DocFreq;
highFreqQueue.Top.term = BytesRef.DeepCopyOf(iterator.Term);
highFreqQueue.UpdateTop();
}
if (lowFreqQueue.Top.freq > iterator.DocFreq)
{
lowFreqQueue.Top.freq = iterator.DocFreq;
lowFreqQueue.Top.term = BytesRef.DeepCopyOf(iterator.Term);
lowFreqQueue.UpdateTop();
}
}
}
int lowFreq = lowFreqQueue.Top.freq;
int highFreq = highFreqQueue.Top.freq;
AssumeTrue(@"unlucky index", highFreq - 1 > lowFreq);
List <TermAndFreq> highTerms = QueueToList(highFreqQueue);
List <TermAndFreq> lowTerms = QueueToList(lowFreqQueue);
IndexSearcher searcher = NewSearcher(reader);
Occur lowFreqOccur = RandomOccur(Random);
BooleanQuery verifyQuery = new BooleanQuery();
CommonTermsQuery cq = new CommonTermsQuery(RandomOccur(Random),
lowFreqOccur, highFreq - 1, Random.NextBoolean());
foreach (TermAndFreq termAndFreq in lowTerms)
{
cq.Add(new Term(field, termAndFreq.term));
verifyQuery.Add(new BooleanClause(new TermQuery(new Term(field,
termAndFreq.term)), lowFreqOccur));
}
foreach (TermAndFreq termAndFreq in highTerms)
{
cq.Add(new Term(field, termAndFreq.term));
}
TopDocs cqSearch = searcher.Search(cq, reader.MaxDoc);
TopDocs verifySearch = searcher.Search(verifyQuery, reader.MaxDoc);
assertEquals(verifySearch.TotalHits, cqSearch.TotalHits);
var hits = new JCG.HashSet <int>();
foreach (ScoreDoc doc in verifySearch.ScoreDocs)
{
hits.Add(doc.Doc);
}
foreach (ScoreDoc doc in cqSearch.ScoreDocs)
{
assertTrue(hits.Remove(doc.Doc));
}
assertTrue(hits.Count == 0);
/*
* need to force merge here since QueryUtils adds checks based
* on leave readers which have different statistics than the top
* level reader if we have more than one segment. This could
* result in a different query / results.
*/
w.ForceMerge(1);
DirectoryReader reader2 = w.GetReader();
QueryUtils.Check(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, cq, NewSearcher(reader2));
reader2.Dispose();
}
finally
{
reader.Dispose();
wrapper.Dispose();
w.Dispose();
dir.Dispose();
}
}
public void TestRandomIndex()
{
Directory dir = NewDirectory();
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
RandomIndexWriter w = new RandomIndexWriter(Random, dir, analyzer, Similarity, TimeZone);
CreateRandomIndex(AtLeast(50), w, Random.NextInt64());
DirectoryReader reader = w.GetReader();
AtomicReader wrapper = SlowCompositeReaderWrapper.Wrap(reader);
string field = @"body";
Terms terms = wrapper.GetTerms(field);
var lowFreqQueue = new AnonymousPriorityQueue(this, 5);
Util.PriorityQueue <TermAndFreq> highFreqQueue = new AnonymousPriorityQueue1(this, 5);
try
{
TermsEnum iterator = terms.GetIterator(null);
while (iterator.Next() != null)
{
if (highFreqQueue.Count < 5)
{
highFreqQueue.Add(new TermAndFreq(BytesRef.DeepCopyOf(iterator.Term), iterator.DocFreq));
lowFreqQueue.Add(new TermAndFreq(BytesRef.DeepCopyOf(iterator.Term), iterator.DocFreq));
}
else
{
if (highFreqQueue.Top.freq < iterator.DocFreq)
{
highFreqQueue.Top.freq = iterator.DocFreq;
highFreqQueue.Top.term = BytesRef.DeepCopyOf(iterator.Term);
highFreqQueue.UpdateTop();
}
if (lowFreqQueue.Top.freq > iterator.DocFreq)
{
lowFreqQueue.Top.freq = iterator.DocFreq;
lowFreqQueue.Top.term = BytesRef.DeepCopyOf(iterator.Term);
lowFreqQueue.UpdateTop();
}
}
}
int lowFreq = lowFreqQueue.Top.freq;
int highFreq = highFreqQueue.Top.freq;
AssumeTrue(@"unlucky index", highFreq - 1 > lowFreq);
List <TermAndFreq> highTerms = QueueToList(highFreqQueue);
List <TermAndFreq> lowTerms = QueueToList(lowFreqQueue);
IndexSearcher searcher = NewSearcher(reader);
Occur lowFreqOccur = RandomOccur(Random);
BooleanQuery verifyQuery = new BooleanQuery();
CommonTermsQuery cq = new CommonTermsQuery(RandomOccur(Random), lowFreqOccur, highFreq - 1, Random.NextBoolean());
foreach (TermAndFreq termAndFreq in lowTerms)
{
cq.Add(new Term(field, termAndFreq.term));
verifyQuery.Add(new BooleanClause(new TermQuery(new Term(field, termAndFreq.term)), lowFreqOccur));
}
foreach (TermAndFreq termAndFreq in highTerms)
{
cq.Add(new Term(field, termAndFreq.term));
}
TopDocs cqSearch = searcher.Search(cq, reader.MaxDoc);
TopDocs verifySearch = searcher.Search(verifyQuery, reader.MaxDoc);
assertEquals(verifySearch.TotalHits, cqSearch.TotalHits);
var hits = new HashSet <int>();
foreach (ScoreDoc doc in verifySearch.ScoreDocs)
{
hits.Add(doc.Doc);
}
foreach (ScoreDoc doc in cqSearch.ScoreDocs)
{
assertTrue(hits.Remove(doc.Doc));
}
assertTrue(hits.Count == 0);
w.ForceMerge(1);
DirectoryReader reader2 = w.GetReader();
QueryUtils.Check(Random, cq, NewSearcher(reader2), Similarity);
reader2.Dispose();
}
finally
{
reader.Dispose();
wrapper.Dispose();
w.Dispose();
dir.Dispose();
}
}
public void TestRandom()
{
int numberOfRuns = TestUtil.NextInt32(Random, 3, 6);
for (int iter = 0; iter < numberOfRuns; iter++)
{
if (VERBOSE)
{
Console.WriteLine(string.Format("TEST: iter={0} total={1}", iter, numberOfRuns));
}
int numDocs = TestUtil.NextInt32(Random, 100, 1000) * RANDOM_MULTIPLIER;
int numGroups = TestUtil.NextInt32(Random, 1, numDocs);
if (VERBOSE)
{
Console.WriteLine("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
}
List <BytesRef> groups = new List <BytesRef>();
for (int i = 0; i < numGroups; i++)
{
string randomValue;
do
{
// B/c of DV based impl we can't see the difference between an empty string and a null value.
// For that reason we don't generate empty string groups.
randomValue = TestUtil.RandomRealisticUnicodeString(Random);
} while ("".Equals(randomValue, StringComparison.Ordinal));
groups.Add(new BytesRef(randomValue));
}
string[] contentStrings = new string[TestUtil.NextInt32(Random, 2, 20)];
if (VERBOSE)
{
Console.WriteLine("TEST: create fake content");
}
for (int contentIDX = 0; contentIDX < contentStrings.Length; contentIDX++)
{
StringBuilder sb = new StringBuilder();
sb.append("real").append(Random.nextInt(3)).append(' ');
int fakeCount = Random.nextInt(10);
for (int fakeIDX = 0; fakeIDX < fakeCount; fakeIDX++)
{
sb.append("fake ");
}
contentStrings[contentIDX] = sb.toString();
if (VERBOSE)
{
Console.WriteLine(" content=" + sb.toString());
}
}
Directory dir = NewDirectory();
RandomIndexWriter w = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)));
bool preFlex = "Lucene3x".Equals(w.IndexWriter.Config.Codec.Name, StringComparison.Ordinal);
bool canUseIDV = !preFlex;
DocValuesType valueType = vts[Random.nextInt(vts.Length)];
Document doc = new Document();
Document docNoGroup = new Document();
Field group = NewStringField("group", "", Field.Store.NO);
doc.Add(group);
Field valuesField = null;
if (canUseIDV)
{
switch (valueType)
{
case DocValuesType.BINARY:
valuesField = new BinaryDocValuesField("group_dv", new BytesRef());
break;
case DocValuesType.SORTED:
valuesField = new SortedDocValuesField("group_dv", new BytesRef());
break;
default:
fail("unhandled type");
break;
}
doc.Add(valuesField);
}
Field sort1 = NewStringField("sort1", "", Field.Store.NO);
doc.Add(sort1);
docNoGroup.Add(sort1);
Field sort2 = NewStringField("sort2", "", Field.Store.NO);
doc.Add(sort2);
docNoGroup.Add(sort2);
Field sort3 = NewStringField("sort3", "", Field.Store.NO);
doc.Add(sort3);
docNoGroup.Add(sort3);
Field content = NewTextField("content", "", Field.Store.NO);
doc.Add(content);
docNoGroup.Add(content);
Int32Field id = new Int32Field("id", 0, Field.Store.NO);
doc.Add(id);
docNoGroup.Add(id);
GroupDoc[] groupDocs = new GroupDoc[numDocs];
for (int i = 0; i < numDocs; i++)
{
BytesRef groupValue;
if (Random.nextInt(24) == 17)
{
// So we test the "doc doesn't have the group'd
// field" case:
groupValue = null;
}
else
{
groupValue = groups[Random.nextInt(groups.size())];
}
GroupDoc groupDoc = new GroupDoc(
i,
groupValue,
groups[Random.nextInt(groups.size())],
groups[Random.nextInt(groups.size())],
new BytesRef(string.Format(CultureInfo.InvariantCulture, "{0:D5}", i)),
contentStrings[Random.nextInt(contentStrings.Length)]
);
if (VERBOSE)
{
Console.WriteLine(" doc content=" + groupDoc.content + " id=" + i + " group=" + (groupDoc.group == null ? "null" : groupDoc.group.Utf8ToString()) + " sort1=" + groupDoc.sort1.Utf8ToString() + " sort2=" + groupDoc.sort2.Utf8ToString() + " sort3=" + groupDoc.sort3.Utf8ToString());
}
groupDocs[i] = groupDoc;
if (groupDoc.group != null)
{
group.SetStringValue(groupDoc.group.Utf8ToString());
if (canUseIDV)
{
valuesField.SetBytesValue(new BytesRef(groupDoc.group.Utf8ToString()));
}
}
sort1.SetStringValue(groupDoc.sort1.Utf8ToString());
sort2.SetStringValue(groupDoc.sort2.Utf8ToString());
sort3.SetStringValue(groupDoc.sort3.Utf8ToString());
content.SetStringValue(groupDoc.content);
id.SetInt32Value(groupDoc.id);
if (groupDoc.group == null)
{
w.AddDocument(docNoGroup);
}
else
{
w.AddDocument(doc);
}
}
DirectoryReader r = w.GetReader();
w.Dispose();
// NOTE: intentional but temporary field cache insanity!
FieldCache.Int32s docIdToFieldId = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
int[] fieldIdToDocID = new int[numDocs];
for (int i = 0; i < numDocs; i++)
{
int fieldId = docIdToFieldId.Get(i);
fieldIdToDocID[fieldId] = i;
}
try
{
IndexSearcher s = NewSearcher(r);
if (typeof(SlowCompositeReaderWrapper).GetTypeInfo().IsAssignableFrom(s.IndexReader.GetType()))
{
canUseIDV = false;
}
else
{
canUseIDV = !preFlex;
}
for (int contentID = 0; contentID < 3; contentID++)
{
ScoreDoc[] hits = s.Search(new TermQuery(new Term("content", "real" + contentID)), numDocs).ScoreDocs;
foreach (ScoreDoc hit in hits)
{
GroupDoc gd = groupDocs[docIdToFieldId.Get(hit.Doc)];
assertTrue(gd.score == 0.0);
gd.score = hit.Score;
int docId = gd.id;
assertEquals(docId, docIdToFieldId.Get(hit.Doc));
}
}
foreach (GroupDoc gd in groupDocs)
{
assertTrue(gd.score != 0.0);
}
for (int searchIter = 0; searchIter < 100; searchIter++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: searchIter=" + searchIter);
}
string searchTerm = "real" + Random.nextInt(3);
bool sortByScoreOnly = Random.nextBoolean();
Sort sortWithinGroup = GetRandomSort(sortByScoreOnly);
AbstractAllGroupHeadsCollector allGroupHeadsCollector = CreateRandomCollector("group", sortWithinGroup, canUseIDV, valueType);
s.Search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
int[] expectedGroupHeads = CreateExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
int[] actualGroupHeads = allGroupHeadsCollector.RetrieveGroupHeads();
// The actual group heads contains Lucene ids. Need to change them into our id value.
for (int i = 0; i < actualGroupHeads.Length; i++)
{
actualGroupHeads[i] = docIdToFieldId.Get(actualGroupHeads[i]);
}
// Allows us the easily iterate and assert the actual and expected results.
Array.Sort(expectedGroupHeads);
Array.Sort(actualGroupHeads);
if (VERBOSE)
{
Console.WriteLine("Collector: " + allGroupHeadsCollector.GetType().Name);
Console.WriteLine("Sort within group: " + sortWithinGroup);
Console.WriteLine("Num group: " + numGroups);
Console.WriteLine("Num doc: " + numDocs);
Console.WriteLine("\n=== Expected: \n");
foreach (int expectedDocId in expectedGroupHeads)
{
GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
string expectedGroup = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.Utf8ToString();
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.Utf8ToString(),
expectedGroupDoc.sort2.Utf8ToString(), expectedGroupDoc.sort3.Utf8ToString(), expectedDocId)
);
}
Console.WriteLine("\n=== Actual: \n");
foreach (int actualDocId in actualGroupHeads)
{
GroupDoc actualGroupDoc = groupDocs[actualDocId];
string actualGroup = actualGroupDoc.group == null ? null : actualGroupDoc.group.Utf8ToString();
Console.WriteLine(
string.Format(CultureInfo.InvariantCulture,
"Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.Utf8ToString(),
actualGroupDoc.sort2.Utf8ToString(), actualGroupDoc.sort3.Utf8ToString(), actualDocId)
);
}
Console.WriteLine("\n===================================================================================");
}
assertArrayEquals(expectedGroupHeads, actualGroupHeads);
}
}
finally
{
QueryUtils.PurgeFieldCache(r);
}
r.Dispose();
dir.Dispose();
}
}
