public override Scorer GetScorer(AtomicReaderContext context, IBits acceptDocs)
{
Debug.Assert(outerInstance.termArrays.Count > 0);
AtomicReader reader = (context.AtomicReader);
IBits liveDocs = acceptDocs;
PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[outerInstance.termArrays.Count];
Terms fieldTerms = reader.GetTerms(outerInstance.field);
if (fieldTerms == null)
{
return(null);
}
// Reuse single TermsEnum below:
TermsEnum termsEnum = fieldTerms.GetIterator(null);
for (int pos = 0; pos < postingsFreqs.Length; pos++)
{
Term[] terms = outerInstance.termArrays[pos];
DocsAndPositionsEnum postingsEnum;
int docFreq;
if (terms.Length > 1)
{
postingsEnum = new UnionDocsAndPositionsEnum(liveDocs, context, terms, termContexts, termsEnum);
// coarse -- this overcounts since a given doc can
// have more than one term:
docFreq = 0;
for (int termIdx = 0; termIdx < terms.Length; termIdx++)
{
Term term = terms[termIdx];
TermState termState = termContexts[term].Get(context.Ord);
if (termState == null)
{
// Term not in reader
continue;
}
termsEnum.SeekExact(term.Bytes, termState);
docFreq += termsEnum.DocFreq;
}
if (docFreq == 0)
{
// None of the terms are in this reader
return(null);
}
}
else
{
Term term = terms[0];
TermState termState = termContexts[term].Get(context.Ord);
if (termState == null)
{
// Term not in reader
return(null);
}
termsEnum.SeekExact(term.Bytes, termState);
postingsEnum = termsEnum.DocsAndPositions(liveDocs, null, DocsAndPositionsFlags.NONE);
if (postingsEnum == null)
{
// term does exist, but has no positions
Debug.Assert(termsEnum.Docs(liveDocs, null, DocsFlags.NONE) != null, "termstate found but no term exists in reader");
throw new InvalidOperationException("field \"" + term.Field + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.Text() + ")");
}
docFreq = termsEnum.DocFreq;
}
postingsFreqs[pos] = new PhraseQuery.PostingsAndFreq(postingsEnum, docFreq, (int)outerInstance.positions[pos], terms);
}
// sort by increasing docFreq order
if (outerInstance.slop == 0)
{
ArrayUtil.TimSort(postingsFreqs);
}
if (outerInstance.slop == 0)
{
ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.GetSimScorer(stats, context));
if (s.noDocs)
{
return(null);
}
else
{
return(s);
}
}
else
{
return(new SloppyPhraseScorer(this, postingsFreqs, outerInstance.slop, similarity.GetSimScorer(stats, context)));
}
}
private IndexIterationContext CreateContext(int nDocs, RandomIndexWriter fromWriter, RandomIndexWriter toWriter,
bool multipleValuesPerDocument, bool scoreDocsInOrder)
{
IndexIterationContext context = new IndexIterationContext();
int numRandomValues = nDocs / 2;
context.RandomUniqueValues = new string[numRandomValues];
ISet <string> trackSet = new HashSet <string>();
context.RandomFrom = new bool[numRandomValues];
for (int i = 0; i < numRandomValues; i++)
{
string uniqueRandomValue;
do
{
uniqueRandomValue = TestUtil.RandomRealisticUnicodeString(Random());
// uniqueRandomValue = TestUtil.randomSimpleString(random);
} while ("".Equals(uniqueRandomValue) || trackSet.Contains(uniqueRandomValue));
// Generate unique values and empty strings aren't allowed.
trackSet.Add(uniqueRandomValue);
context.RandomFrom[i] = Random().NextBoolean();
context.RandomUniqueValues[i] = uniqueRandomValue;
}
RandomDoc[] docs = new RandomDoc[nDocs];
for (int i = 0; i < nDocs; i++)
{
string id = Convert.ToString(i);
int randomI = Random().Next(context.RandomUniqueValues.Length);
string value = context.RandomUniqueValues[randomI];
Document document = new Document();
document.Add(NewTextField(Random(), "id", id, Field.Store.NO));
document.Add(NewTextField(Random(), "value", value, Field.Store.NO));
bool from = context.RandomFrom[randomI];
int numberOfLinkValues = multipleValuesPerDocument ? 2 + Random().Next(10) : 1;
docs[i] = new RandomDoc(id, numberOfLinkValues, value, from);
for (int j = 0; j < numberOfLinkValues; j++)
{
string linkValue = context.RandomUniqueValues[Random().Next(context.RandomUniqueValues.Length)];
docs[i].LinkValues.Add(linkValue);
if (from)
{
if (!context.FromDocuments.ContainsKey(linkValue))
{
context.FromDocuments[linkValue] = new List <RandomDoc>();
}
if (!context.RandomValueFromDocs.ContainsKey(value))
{
context.RandomValueFromDocs[value] = new List <RandomDoc>();
}
context.FromDocuments[linkValue].Add(docs[i]);
context.RandomValueFromDocs[value].Add(docs[i]);
document.Add(NewTextField(Random(), "from", linkValue, Field.Store.NO));
}
else
{
if (!context.ToDocuments.ContainsKey(linkValue))
{
context.ToDocuments[linkValue] = new List <RandomDoc>();
}
if (!context.RandomValueToDocs.ContainsKey(value))
{
context.RandomValueToDocs[value] = new List <RandomDoc>();
}
context.ToDocuments[linkValue].Add(docs[i]);
context.RandomValueToDocs[value].Add(docs[i]);
document.Add(NewTextField(Random(), "to", linkValue, Field.Store.NO));
}
}
RandomIndexWriter w;
if (from)
{
w = fromWriter;
}
else
{
w = toWriter;
}
w.AddDocument(document);
if (Random().Next(10) == 4)
{
w.Commit();
}
if (VERBOSE)
{
Console.WriteLine("Added document[" + docs[i].Id + "]: " + document);
}
}
// Pre-compute all possible hits for all unique random values. On top of this also compute all possible score for
// any ScoreMode.
IndexSearcher fromSearcher = NewSearcher(fromWriter.Reader);
IndexSearcher toSearcher = NewSearcher(toWriter.Reader);
for (int i = 0; i < context.RandomUniqueValues.Length; i++)
{
string uniqueRandomValue = context.RandomUniqueValues[i];
string fromField;
string toField;
IDictionary <string, IDictionary <int, JoinScore> > queryVals;
if (context.RandomFrom[i])
{
fromField = "from";
toField = "to";
queryVals = context.FromHitsToJoinScore;
}
else
{
fromField = "to";
toField = "from";
queryVals = context.ToHitsToJoinScore;
}
IDictionary <BytesRef, JoinScore> joinValueToJoinScores = new Dictionary <BytesRef, JoinScore>();
if (multipleValuesPerDocument)
{
fromSearcher.Search(new TermQuery(new Term("value", uniqueRandomValue)),
new CollectorAnonymousInnerClassHelper3(this, context, fromField, joinValueToJoinScores));
}
else
{
fromSearcher.Search(new TermQuery(new Term("value", uniqueRandomValue)),
new CollectorAnonymousInnerClassHelper4(this, context, fromField, joinValueToJoinScores));
}
IDictionary <int, JoinScore> docToJoinScore = new Dictionary <int, JoinScore>();
if (multipleValuesPerDocument)
{
if (scoreDocsInOrder)
{
AtomicReader slowCompositeReader = SlowCompositeReaderWrapper.Wrap(toSearcher.IndexReader);
Terms terms = slowCompositeReader.Terms(toField);
if (terms != null)
{
DocsEnum docsEnum = null;
TermsEnum termsEnum = null;
SortedSet <BytesRef> joinValues =
new SortedSet <BytesRef>(BytesRef.UTF8SortedAsUnicodeComparer);
joinValues.AddAll(joinValueToJoinScores.Keys);
foreach (BytesRef joinValue in joinValues)
{
termsEnum = terms.Iterator(termsEnum);
if (termsEnum.SeekExact(joinValue))
{
docsEnum = termsEnum.Docs(slowCompositeReader.LiveDocs, docsEnum, DocsEnum.FLAG_NONE);
JoinScore joinScore = joinValueToJoinScores[joinValue];
for (int doc = docsEnum.NextDoc();
doc != DocIdSetIterator.NO_MORE_DOCS;
doc = docsEnum.NextDoc())
{
// First encountered join value determines the score.
// Something to keep in mind for many-to-many relations.
if (!docToJoinScore.ContainsKey(doc))
{
docToJoinScore[doc] = joinScore;
}
}
}
}
}
}
else
{
toSearcher.Search(new MatchAllDocsQuery(),
new CollectorAnonymousInnerClassHelper5(this, context, toField, joinValueToJoinScores,
docToJoinScore));
}
}
else
{
toSearcher.Search(new MatchAllDocsQuery(),
new CollectorAnonymousInnerClassHelper6(this, context, toField, joinValueToJoinScores,
docToJoinScore));
}
queryVals[uniqueRandomValue] = docToJoinScore;
}
fromSearcher.IndexReader.Dispose();
toSearcher.IndexReader.Dispose();
return(context);
}
// algorithm: treat sentence snippets as miniature documents
// we can intersect these with the postings lists via BreakIterator.preceding(offset),s
// score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq))
private Passage[] HighlightDoc(string field, BytesRef[] terms, int contentLength, BreakIterator bi, int doc,
TermsEnum termsEnum, DocsAndPositionsEnum[] postings, int n)
{
PassageScorer scorer = GetScorer(field);
if (scorer == null)
{
throw new NullReferenceException("PassageScorer cannot be null");
}
JCG.PriorityQueue <OffsetsEnum> pq = new JCG.PriorityQueue <OffsetsEnum>();
float[] weights = new float[terms.Length];
// initialize postings
for (int i = 0; i < terms.Length; i++)
{
DocsAndPositionsEnum de = postings[i];
int pDoc;
if (de == EMPTY)
{
continue;
}
else if (de == null)
{
postings[i] = EMPTY; // initially
if (!termsEnum.SeekExact(terms[i]))
{
continue; // term not found
}
de = postings[i] = termsEnum.DocsAndPositions(null, null, DocsAndPositionsFlags.OFFSETS);
if (de == null)
{
// no positions available
throw new ArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
}
pDoc = de.Advance(doc);
}
else
{
pDoc = de.DocID;
if (pDoc < doc)
{
pDoc = de.Advance(doc);
}
}
if (doc == pDoc)
{
weights[i] = scorer.Weight(contentLength, de.Freq);
de.NextPosition();
pq.Add(new OffsetsEnum(de, i));
}
}
pq.Add(new OffsetsEnum(EMPTY, int.MaxValue)); // a sentinel for termination
JCG.PriorityQueue <Passage> passageQueue = new JCG.PriorityQueue <Passage>(n, new HighlightDocComparerAnonymousHelper1());
Passage current = new Passage();
while (pq.TryDequeue(out OffsetsEnum off))
{
DocsAndPositionsEnum dp = off.dp;
int start = dp.StartOffset;
if (start == -1)
{
throw new ArgumentException("field '" + field + "' was indexed without offsets, cannot highlight");
}
int end = dp.EndOffset;
// LUCENE-5166: this hit would span the content limit... however more valid
// hits may exist (they are sorted by start). so we pretend like we never
// saw this term, it won't cause a passage to be added to passageQueue or anything.
Debug.Assert(EMPTY.StartOffset == int.MaxValue);
if (start < contentLength && end > contentLength)
{
continue;
}
if (start >= current.endOffset)
{
if (current.startOffset >= 0)
{
// finalize current
current.score *= scorer.Norm(current.startOffset);
// new sentence: first add 'current' to queue
if (passageQueue.Count == n && current.score < passageQueue.Peek().score)
{
current.Reset(); // can't compete, just reset it
}
else
{
passageQueue.Enqueue(current);
if (passageQueue.Count > n)
{
current = passageQueue.Dequeue();
current.Reset();
}
else
{
current = new Passage();
}
}
}
// if we exceed limit, we are done
if (start >= contentLength)
{
Passage[] passages = passageQueue.ToArray();
foreach (Passage p in passages)
{
p.Sort();
}
// sort in ascending order
ArrayUtil.TimSort(passages, new HighlightDocComparerAnonymousHelper2());
return(passages);
}
// advance breakiterator
Debug.Assert(BreakIterator.Done < 0);
current.startOffset = Math.Max(bi.Preceding(start + 1), 0);
current.endOffset = Math.Min(bi.Next(), contentLength);
}
int tf = 0;
while (true)
{
tf++;
BytesRef term = terms[off.id];
if (term == null)
{
// multitermquery match, pull from payload
term = off.dp.GetPayload();
Debug.Assert(term != null);
}
current.AddMatch(start, end, term);
if (off.pos == dp.Freq)
{
break; // removed from pq
}
else
{
off.pos++;
dp.NextPosition();
start = dp.StartOffset;
end = dp.EndOffset;
}
if (start >= current.endOffset || end > contentLength)
{
pq.Enqueue(off);
break;
}
}
current.score += weights[off.id] * scorer.Tf(tf, current.endOffset - current.startOffset);
}
// Dead code but compiler disagrees:
Debug.Assert(false);
return(null);
}
private void AssertTermsSeeking(Terms leftTerms, Terms rightTerms)
{
TermsEnum leftEnum = null;
TermsEnum rightEnum = null;
// just an upper bound
int numTests = AtLeast(20);
Random random = Random();
// collect this number of terms from the left side
HashSet <BytesRef> tests = new HashSet <BytesRef>();
int numPasses = 0;
while (numPasses < 10 && tests.Count < numTests)
{
leftEnum = leftTerms.Iterator(leftEnum);
BytesRef term = null;
while ((term = leftEnum.Next()) != null)
{
int code = random.Next(10);
if (code == 0)
{
// the term
tests.Add(BytesRef.DeepCopyOf(term));
}
else if (code == 1)
{
// truncated subsequence of term
term = BytesRef.DeepCopyOf(term);
if (term.Length > 0)
{
// truncate it
term.Length = random.Next(term.Length);
}
}
else if (code == 2)
{
// term, but ensure a non-zero offset
var newbytes = new byte[term.Length + 5];
Array.Copy(term.Bytes, term.Offset, newbytes, 5, term.Length);
tests.Add(new BytesRef(newbytes, 5, term.Length));
}
}
numPasses++;
}
List <BytesRef> shuffledTests = new List <BytesRef>(tests);
shuffledTests = (List <BytesRef>)CollectionsHelper.Shuffle(shuffledTests);
foreach (BytesRef b in shuffledTests)
{
leftEnum = leftTerms.Iterator(leftEnum);
rightEnum = rightTerms.Iterator(rightEnum);
Assert.AreEqual(leftEnum.SeekExact(b), rightEnum.SeekExact(b));
Assert.AreEqual(leftEnum.SeekExact(b), rightEnum.SeekExact(b));
SeekStatus leftStatus;
SeekStatus rightStatus;
leftStatus = leftEnum.SeekCeil(b);
rightStatus = rightEnum.SeekCeil(b);
Assert.AreEqual(leftStatus, rightStatus);
if (leftStatus != SeekStatus.END)
{
Assert.AreEqual(leftEnum.Term(), rightEnum.Term());
}
leftStatus = leftEnum.SeekCeil(b);
rightStatus = rightEnum.SeekCeil(b);
Assert.AreEqual(leftStatus, rightStatus);
if (leftStatus != SeekStatus.END)
{
Assert.AreEqual(leftEnum.Term(), rightEnum.Term());
}
}
}
public override Scorer GetScorer(AtomicReaderContext context, IBits acceptDocs)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(outerInstance.terms.Count > 0);
}
AtomicReader reader = context.AtomicReader;
IBits liveDocs = acceptDocs;
PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[outerInstance.terms.Count];
Terms fieldTerms = reader.GetTerms(outerInstance.field);
if (fieldTerms == null)
{
return(null);
}
// Reuse single TermsEnum below:
TermsEnum te = fieldTerms.GetEnumerator();
for (int i = 0; i < outerInstance.terms.Count; i++)
{
Term t = outerInstance.terms[i];
TermState state = states[i].Get(context.Ord);
if (state == null) // term doesnt exist in this segment
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(TermNotInReader(reader, t), "no termstate found but term exists in reader");
}
return(null);
}
te.SeekExact(t.Bytes, state);
DocsAndPositionsEnum postingsEnum = te.DocsAndPositions(liveDocs, null, DocsAndPositionsFlags.NONE);
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null)
{
if (Debugging.AssertsEnabled)
{
Debugging.Assert(te.SeekExact(t.Bytes), "termstate found but no term exists in reader");
}
// term does exist, but has no positions
throw new InvalidOperationException("field \"" + t.Field + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.Text() + ")");
}
postingsFreqs[i] = new PostingsAndFreq(postingsEnum, te.DocFreq, (int)outerInstance.positions[i], t);
}
// sort by increasing docFreq order
if (outerInstance.slop == 0)
{
ArrayUtil.TimSort(postingsFreqs);
}
if (outerInstance.slop == 0) // optimize exact case
{
ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.GetSimScorer(stats, context));
if (s.noDocs)
{
return(null);
}
else
{
return(s);
}
}
else
{
return(new SloppyPhraseScorer(this, postingsFreqs, outerInstance.slop, similarity.GetSimScorer(stats, context)));
}
}
public override Scorer Scorer(AtomicReaderContext context, Bits acceptDocs)
{
Debug.Assert(OuterInstance.Terms_Renamed.Count > 0);
AtomicReader reader = context.AtomicReader;
Bits liveDocs = acceptDocs;
PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[OuterInstance.Terms_Renamed.Count];
Terms fieldTerms = reader.Terms(OuterInstance.Field);
if (fieldTerms == null)
{
return(null);
}
// Reuse single TermsEnum below:
TermsEnum te = fieldTerms.Iterator(null);
for (int i = 0; i < OuterInstance.Terms_Renamed.Count; i++)
{
Term t = OuterInstance.Terms_Renamed[i];
TermState state = States[i].Get(context.Ord);
if (state == null) // term doesnt exist in this segment
{
Debug.Assert(TermNotInReader(reader, t), "no termstate found but term exists in reader");
return(null);
}
te.SeekExact(t.Bytes(), state);
DocsAndPositionsEnum postingsEnum = te.DocsAndPositions(liveDocs, null, DocsEnum.FLAG_NONE);
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null)
{
Debug.Assert(te.SeekExact(t.Bytes()), "termstate found but no term exists in reader");
// term does exist, but has no positions
throw new InvalidOperationException("field \"" + t.Field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.Text() + ")");
}
postingsFreqs[i] = new PostingsAndFreq(postingsEnum, te.DocFreq(), (int)OuterInstance.Positions_Renamed[i], t);
}
// sort by increasing docFreq order
if (OuterInstance.Slop_Renamed == 0)
{
ArrayUtil.TimSort(postingsFreqs);
}
if (OuterInstance.Slop_Renamed == 0) // optimize exact case
{
ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, Similarity.DoSimScorer(Stats, context));
if (s.NoDocs)
{
return(null);
}
else
{
return(s);
}
}
else
{
return(new SloppyPhraseScorer(this, postingsFreqs, OuterInstance.Slop_Renamed, Similarity.DoSimScorer(Stats, context)));
}
}
/// <exception cref="System.IO.IOException"></exception>
internal SegmentResult(int[] counts, int total, int missingCountIndex, TermsEnum
tenum, int startFacetOrd, int endFacetOrd) : base(counts, total - counts[missingCountIndex
], counts[missingCountIndex], endFacetOrd == missingCountIndex + 1 ? missingCountIndex
: endFacetOrd)
{
this.tenum = tenum;
this.mergePos = startFacetOrd;
if (tenum != null)
{
tenum.SeekExact(mergePos);
mergeTerm = tenum.Term();
}
}
/// <summary>
/// Look up the given category in the cache and/or the on-disk storage,
/// returning the category's ordinal, or a negative number in case the
/// category does not yet exist in the taxonomy.
/// </summary>
protected virtual int FindCategory(FacetLabel categoryPath)
{
lock (this)
{
// If we can find the category in the cache, or we know the cache is
// complete, we can return the response directly from it
int res = cache.Get(categoryPath);
if (res >= 0 || cacheIsComplete)
{
return(res);
}
cacheMisses.IncrementAndGet();
// After a few cache misses, it makes sense to read all the categories
// from disk and into the cache. The reason not to do this on the first
// cache miss (or even when opening the writer) is that it will
// significantly slow down the case when a taxonomy is opened just to
// add one category. The idea only spending a long time on reading
// after enough time was spent on cache misses is known as an "online
// algorithm".
PerhapsFillCache();
res = cache.Get(categoryPath);
if (res >= 0 || cacheIsComplete)
{
// if after filling the cache from the info on disk, the category is in it
// or the cache is complete, return whatever cache.get returned.
return(res);
}
// if we get here, it means the category is not in the cache, and it is not
// complete, and therefore we must look for the category on disk.
// We need to get an answer from the on-disk index.
InitReaderManager();
int doc = -1;
DirectoryReader reader = readerManager.Acquire();
try
{
BytesRef catTerm = new BytesRef(FacetsConfig.PathToString(categoryPath.Components, categoryPath.Length));
TermsEnum termsEnum = null; // reuse
DocsEnum docs = null; // reuse
foreach (AtomicReaderContext ctx in reader.Leaves)
{
Terms terms = ctx.AtomicReader.GetTerms(Consts.FULL);
if (terms != null)
{
termsEnum = terms.GetIterator(termsEnum);
if (termsEnum.SeekExact(catTerm))
{
// liveDocs=null because the taxonomy has no deletes
docs = termsEnum.Docs(null, docs, 0); // freqs not required
// if the term was found, we know it has exactly one document.
doc = docs.NextDoc() + ctx.DocBase;
break;
}
}
}
}
finally
{
readerManager.Release(reader);
}
if (doc > 0)
{
AddToCache(categoryPath, doc);
}
return(doc);
}
}