public void TestConcurrentSpan()
{
String TEXT = "the fox jumped";
Directory directory = NewDirectory();
IndexWriter indexWriter = new IndexWriter(directory,
NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)));
try
{
Document document = new Document();
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.StoreTermVectorOffsets = (true);
customType.StoreTermVectorPositions = (true);
customType.StoreTermVectors = (true);
document.Add(new Field(FIELD, new TokenStreamConcurrent(), customType));
indexWriter.AddDocument(document);
}
finally
{
indexWriter.Dispose();
}
IndexReader indexReader = DirectoryReader.Open(directory);
try
{
assertEquals(1, indexReader.NumDocs);
IndexSearcher indexSearcher = NewSearcher(indexReader);
Query phraseQuery = new SpanNearQuery(new SpanQuery[] {
new SpanTermQuery(new Term(FIELD, "fox")),
new SpanTermQuery(new Term(FIELD, "jumped"))
}, 0, true);
FixedBitSet bitset = new FixedBitSet(indexReader.MaxDoc);
indexSearcher.Search(phraseQuery, new ConcurrentSpanCollectorAnonymousHelper(this, bitset));
assertEquals(1, bitset.Cardinality());
int maxDoc = indexReader.MaxDoc;
Highlighter highlighter = new Highlighter(
new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
new QueryScorer(phraseQuery));
for (int position = bitset.NextSetBit(0); position >= 0 && position < maxDoc - 1; position = bitset
.NextSetBit(position + 1))
{
assertEquals(0, position);
TokenStream tokenStream = TokenSources.GetTokenStream(
indexReader.GetTermVector(position,
FIELD), false);
assertEquals(highlighter.GetBestFragment(new TokenStreamConcurrent(),
TEXT), highlighter.GetBestFragment(tokenStream, TEXT));
}
}
finally
{
indexReader.Dispose();
directory.Dispose();
}
}
void doNextSetBit(BitArray a, FixedBitSet b)
{
int aa = -1, bb = -1;
do
{
aa = a.NextSetBit(aa + 1);
bb = bb < b.Length() - 1 ? b.NextSetBit(bb + 1) : -1;
Assert.AreEqual(aa, bb);
} while (aa >= 0);
}
void doNextSetBit(BitArray a, FixedBitSet b)
{
int aa = -1, bb = -1;
do
{
aa = a.NextSetBit(aa + 1);
bb = bb < b.Length() - 1 ? b.NextSetBit(bb + 1) : -1;
Assert.AreEqual(aa, bb);
} while (aa >= 0);
}
/// <param name="targetMaxSaturation">
/// A number between 0 and 1 describing the % of bits that would ideally be set in the result.
/// Lower values have better accuracy but require more space.
/// </param>
/// <return>A smaller <see cref="FuzzySet"/> or <c>null</c> if the current set is already over-saturated.</return>
public virtual FuzzySet Downsize(float targetMaxSaturation)
{
var numBitsSet = _filter.Cardinality();
FixedBitSet rightSizedBitSet;
var rightSizedBitSetSize = _bloomSize;
//Hopefully find a smaller size bitset into which we can project accumulated values while maintaining desired saturation level
for (int i = 0; i < _usableBitSetSizes.Length; i++)
{
int candidateBitsetSize = _usableBitSetSizes[i];
float candidateSaturation = (float)numBitsSet
/ (float)candidateBitsetSize;
if (candidateSaturation <= targetMaxSaturation)
{
rightSizedBitSetSize = candidateBitsetSize;
break;
}
}
// Re-project the numbers to a smaller space if necessary
if (rightSizedBitSetSize < _bloomSize)
{
// Reset the choice of bitset to the smaller version
rightSizedBitSet = new FixedBitSet(rightSizedBitSetSize + 1);
// Map across the bits from the large set to the smaller one
var bitIndex = 0;
do
{
bitIndex = _filter.NextSetBit(bitIndex);
if (bitIndex < 0)
{
continue;
}
// Project the larger number into a smaller one effectively
// modulo-ing by using the target bitset size as a mask
var downSizedBitIndex = bitIndex & rightSizedBitSetSize;
rightSizedBitSet.Set(downSizedBitIndex);
bitIndex++;
} while ((bitIndex >= 0) && (bitIndex <= _bloomSize));
}
else
{
return(null);
}
return(new FuzzySet(rightSizedBitSet, rightSizedBitSetSize, _hashFunction));
}
internal virtual int Parent(int doc)
{
return(parentBits.NextSetBit(doc));
}
public override int NextDoc()
{
//System.out.println("Q.nextDoc() nextChildDoc=" + nextChildDoc);
// Loop until we hit a parentDoc that's accepted
while (true)
{
if (_nextChildDoc == NO_MORE_DOCS)
{
//System.out.println(" end");
return(_parentDoc = NO_MORE_DOCS);
}
// Gather all children sharing the same parent as
// nextChildDoc
_parentDoc = _parentBits.NextSetBit(_nextChildDoc);
// Parent & child docs are supposed to be
// orthogonal:
if (_nextChildDoc == _parentDoc)
{
throw IllegalStateException.Create("child query must only match non-parent docs, but parent docID=" + _nextChildDoc + " matched childScorer=" + _childScorer.GetType());
}
//System.out.println(" parentDoc=" + parentDoc);
if (Debugging.AssertsEnabled)
{
Debugging.Assert(_parentDoc != -1);
}
//System.out.println(" nextChildDoc=" + nextChildDoc);
if (_acceptDocs != null && !_acceptDocs.Get(_parentDoc))
{
// Parent doc not accepted; skip child docs until
// we hit a new parent doc:
do
{
_nextChildDoc = _childScorer.NextDoc();
} while (_nextChildDoc < _parentDoc);
// Parent & child docs are supposed to be
// orthogonal:
if (_nextChildDoc == _parentDoc)
{
throw IllegalStateException.Create("child query must only match non-parent docs, but parent docID=" + _nextChildDoc + " matched childScorer=" + _childScorer.GetType());
}
continue;
}
float totalScore = 0;
float maxScore = float.NegativeInfinity;
_childDocUpto = 0;
_parentFreq = 0;
do
{
//System.out.println(" c=" + nextChildDoc);
if (_pendingChildDocs != null && _pendingChildDocs.Length == _childDocUpto)
{
_pendingChildDocs = ArrayUtil.Grow(_pendingChildDocs);
}
if (_pendingChildScores != null && _scoreMode != ScoreMode.None && _pendingChildScores.Length == _childDocUpto)
{
_pendingChildScores = ArrayUtil.Grow(_pendingChildScores);
}
if (_pendingChildDocs != null)
{
_pendingChildDocs[_childDocUpto] = _nextChildDoc;
}
if (_scoreMode != ScoreMode.None)
{
// TODO: specialize this into dedicated classes per-scoreMode
float childScore = _childScorer.GetScore();
int childFreq = _childScorer.Freq;
if (_pendingChildScores != null)
{
_pendingChildScores[_childDocUpto] = childScore;
}
maxScore = Math.Max(childScore, maxScore);
totalScore += childScore;
_parentFreq += childFreq;
}
_childDocUpto++;
_nextChildDoc = _childScorer.NextDoc();
} while (_nextChildDoc < _parentDoc);
// Parent & child docs are supposed to be
// orthogonal:
if (_nextChildDoc == _parentDoc)
{
throw IllegalStateException.Create("child query must only match non-parent docs, but parent docID=" + _nextChildDoc + " matched childScorer=" + _childScorer.GetType());
}
switch (_scoreMode)
{
case ScoreMode.Avg:
_parentScore = totalScore / _childDocUpto;
break;
case ScoreMode.Max:
_parentScore = maxScore;
break;
case ScoreMode.Total:
_parentScore = totalScore;
break;
case ScoreMode.None:
break;
}
//System.out.println(" return parentDoc=" + parentDoc + " childDocUpto=" + childDocUpto);
return(_parentDoc);
}
}
public void Test()
{
RandomIndexWriter writer;
DirectoryReader indexReader;
int numParents = AtLeast(200);
IndexWriterConfig cfg = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
cfg.SetMergePolicy(NewLogMergePolicy());
using (writer = new RandomIndexWriter(Random(), NewDirectory(), cfg))
{
Document parentDoc = new Document();
NumericDocValuesField parentVal = new NumericDocValuesField("parent_val", 0L);
parentDoc.Add(parentVal);
StringField parent = new StringField("parent", "true", Field.Store.YES);
parentDoc.Add(parent);
for (int i = 0; i < numParents; ++i)
{
List <Document> documents = new List <Document>();
int numChildren = Random().nextInt(10);
for (int j = 0; j < numChildren; ++j)
{
Document childDoc = new Document();
childDoc.Add(new NumericDocValuesField("child_val", Random().nextInt(5)));
documents.Add(childDoc);
}
parentVal.SetInt64Value(Random().nextInt(50));
documents.Add(parentDoc);
writer.AddDocuments(documents);
}
writer.ForceMerge(1);
indexReader = writer.Reader;
}
AtomicReader reader = GetOnlySegmentReader(indexReader);
Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
FixedBitSet parentBits = (FixedBitSet)parentsFilter.GetDocIdSet(reader.AtomicContext, null);
NumericDocValues parentValues = reader.GetNumericDocValues("parent_val");
NumericDocValues childValues = reader.GetNumericDocValues("child_val");
Sort parentSort = new Sort(new SortField("parent_val", SortFieldType.INT64));
Sort childSort = new Sort(new SortField("child_val", SortFieldType.INT64));
Sort sort = new Sort(new SortField("custom", new BlockJoinComparerSource(parentsFilter, parentSort, childSort)));
Sorter sorter = new Sorter(sort);
Sorter.DocMap docMap = sorter.Sort(reader);
assertEquals(reader.MaxDoc, docMap.Count);
int[] children = new int[1];
int numChildren2 = 0;
int previousParent = -1;
for (int i = 0; i < docMap.Count; ++i)
{
int oldID = docMap.NewToOld(i);
if (parentBits.Get(oldID))
{
// check that we have the right children
for (int j = 0; j < numChildren2; ++j)
{
assertEquals(oldID, parentBits.NextSetBit(children[j]));
}
// check that children are sorted
for (int j = 1; j < numChildren2; ++j)
{
int doc1 = children[j - 1];
int doc2 = children[j];
if (childValues.Get(doc1) == childValues.Get(doc2))
{
assertTrue(doc1 < doc2); // sort is stable
}
else
{
assertTrue(childValues.Get(doc1) < childValues.Get(doc2));
}
}
// check that parents are sorted
if (previousParent != -1)
{
if (parentValues.Get(previousParent) == parentValues.Get(oldID))
{
assertTrue(previousParent < oldID);
}
else
{
assertTrue(parentValues.Get(previousParent) < parentValues.Get(oldID));
}
}
// reset
previousParent = oldID;
numChildren2 = 0;
}
else
{
children = ArrayUtil.Grow(children, numChildren2 + 1);
children[numChildren2++] = oldID;
}
}
indexReader.Dispose();
writer.w.Directory.Dispose();
}
/// <summary>
/// Used when drill downs are highly constraining vs
/// baseQuery.
/// </summary>
private void DoDrillDownAdvanceScoring(ICollector collector, DocIdSetIterator[] disis, ICollector[] sidewaysCollectors)
{
int maxDoc = context.Reader.MaxDoc;
int numDims = dims.Length;
//if (DEBUG) {
// System.out.println(" doDrillDownAdvanceScoring");
//}
// TODO: maybe a class like BS, instead of parallel arrays
int[] filledSlots = new int[CHUNK];
int[] docIDs = new int[CHUNK];
float[] scores = new float[CHUNK];
int[] missingDims = new int[CHUNK];
int[] counts = new int[CHUNK];
docIDs[0] = -1;
int nextChunkStart = CHUNK;
FixedBitSet seen = new FixedBitSet(CHUNK);
while (true)
{
//if (DEBUG) {
// System.out.println("\ncycle nextChunkStart=" + nextChunkStart + " docIds[0]=" + docIDs[0]);
//}
// First dim:
//if (DEBUG) {
// System.out.println(" dim0");
//}
DocIdSetIterator disi = disis[0];
if (disi != null)
{
int docID = disi.DocID;
while (docID < nextChunkStart)
{
int slot = docID & MASK;
if (docIDs[slot] != docID)
{
seen.Set(slot);
// Mark slot as valid:
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " id=" + context.reader().document(docID).get("id"));
//}
docIDs[slot] = docID;
missingDims[slot] = 1;
counts[slot] = 1;
}
docID = disi.NextDoc();
}
}
// Second dim:
//if (DEBUG) {
// System.out.println(" dim1");
//}
disi = disis[1];
if (disi != null)
{
int docID = disi.DocID;
while (docID < nextChunkStart)
{
int slot = docID & MASK;
if (docIDs[slot] != docID)
{
// Mark slot as valid:
seen.Set(slot);
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " missingDim=0 id=" + context.reader().document(docID).get("id"));
//}
docIDs[slot] = docID;
missingDims[slot] = 0;
counts[slot] = 1;
}
else
{
// TODO: single-valued dims will always be true
// below; we could somehow specialize
if (missingDims[slot] >= 1)
{
missingDims[slot] = 2;
counts[slot] = 2;
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " missingDim=2 id=" + context.reader().document(docID).get("id"));
//}
}
else
{
counts[slot] = 1;
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " missingDim=" + missingDims[slot] + " id=" + context.reader().document(docID).get("id"));
//}
}
}
docID = disi.NextDoc();
}
}
// After this we can "upgrade" to conjunction, because
// any doc not seen by either dim 0 or dim 1 cannot be
// a hit or a near miss:
//if (DEBUG) {
// System.out.println(" baseScorer");
//}
// Fold in baseScorer, using advance:
int filledCount = 0;
int slot0 = 0;
while (slot0 < CHUNK && (slot0 = seen.NextSetBit(slot0)) != -1)
{
int ddDocID = docIDs[slot0];
if (Debugging.AssertsEnabled)
{
Debugging.Assert(ddDocID != -1);
}
int baseDocID = baseScorer.DocID;
if (baseDocID < ddDocID)
{
baseDocID = baseScorer.Advance(ddDocID);
}
if (baseDocID == ddDocID)
{
//if (DEBUG) {
// System.out.println(" keep docID=" + ddDocID + " id=" + context.reader().document(ddDocID).get("id"));
//}
scores[slot0] = baseScorer.GetScore();
filledSlots[filledCount++] = slot0;
counts[slot0]++;
}
else
{
//if (DEBUG) {
// System.out.println(" no docID=" + ddDocID + " id=" + context.reader().document(ddDocID).get("id"));
//}
docIDs[slot0] = -1;
// TODO: we could jump slot0 forward to the
// baseDocID ... but we'd need to set docIDs for
// intervening slots to -1
}
slot0++;
}
seen.Clear(0, CHUNK);
if (filledCount == 0)
{
if (nextChunkStart >= maxDoc)
{
break;
}
nextChunkStart += CHUNK;
continue;
}
// TODO: factor this out & share w/ union scorer,
// except we start from dim=2 instead:
for (int dim = 2; dim < numDims; dim++)
{
//if (DEBUG) {
// System.out.println(" dim=" + dim + " [" + dims[dim].dim + "]");
//}
disi = disis[dim];
if (disi != null)
{
int docID = disi.DocID;
while (docID < nextChunkStart)
{
int slot = docID & MASK;
if (docIDs[slot] == docID && counts[slot] >= dim)
{
// TODO: single-valued dims will always be true
// below; we could somehow specialize
if (missingDims[slot] >= dim)
{
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " count=" + (dim+2));
//}
missingDims[slot] = dim + 1;
counts[slot] = dim + 2;
}
else
{
//if (DEBUG) {
// System.out.println(" set docID=" + docID + " missing count=" + (dim+1));
//}
counts[slot] = dim + 1;
}
}
// TODO: sometimes use advance?
docID = disi.NextDoc();
}
}
}
// Collect:
//if (DEBUG) {
// System.out.println(" now collect: " + filledCount + " hits");
//}
for (int i = 0; i < filledCount; i++)
{
int slot = filledSlots[i];
collectDocID = docIDs[slot];
collectScore = scores[slot];
//if (DEBUG) {
// System.out.println(" docID=" + docIDs[slot] + " count=" + counts[slot]);
//}
if (counts[slot] == 1 + numDims)
{
CollectHit(collector, sidewaysCollectors);
}
else if (counts[slot] == numDims)
{
CollectNearMiss(sidewaysCollectors[missingDims[slot]]);
}
}
if (nextChunkStart >= maxDoc)
{
break;
}
nextChunkStart += CHUNK;
}
}