/// <summary>
/// a constructor.
/// </summary>
/// <param name="fieldTermStack">FieldTermStack object</param>
/// <param name="fieldQuery">FieldQuery object</param>
/// <param name="phraseLimit">maximum size of phraseList</param>
public FieldPhraseList(FieldTermStack fieldTermStack, FieldQuery fieldQuery, int phraseLimit)
{
String field = fieldTermStack.FieldName;
LinkedList<TermInfo> phraseCandidate = new LinkedList<TermInfo>();
QueryPhraseMap currMap = null;
QueryPhraseMap nextMap = null;
while (!fieldTermStack.IsEmpty() && (phraseList.Count < phraseLimit) )
{
phraseCandidate.Clear();
TermInfo ti = fieldTermStack.Pop();
currMap = fieldQuery.GetFieldTermMap(field, ti.Text);
// if not found, discard top TermInfo from stack, then try next element
if (currMap == null) continue;
// if found, search the longest phrase
phraseCandidate.AddLast(ti);
while (true)
{
ti = fieldTermStack.Pop();
nextMap = null;
if (ti != null)
nextMap = currMap.GetTermMap(ti.Text);
if (ti == null || nextMap == null)
{
if (ti != null)
fieldTermStack.Push(ti);
if (currMap.IsValidTermOrPhrase(new List<TermInfo>(phraseCandidate)))
{
AddIfNoOverlap(new WeightedPhraseInfo(phraseCandidate, currMap.Boost, currMap.TermOrPhraseNumber));
}
else
{
while (phraseCandidate.Count > 1)
{
TermInfo last = phraseCandidate.Last.Value;
phraseCandidate.RemoveLast();
fieldTermStack.Push(last);
currMap = fieldQuery.SearchPhrase(field, new List<TermInfo>(phraseCandidate));
if (currMap != null)
{
AddIfNoOverlap(new WeightedPhraseInfo(phraseCandidate, currMap.Boost, currMap.TermOrPhraseNumber));
break;
}
}
}
break;
}
else
{
phraseCandidate.AddLast(ti);
currMap = nextMap;
}
}
}
}
public void TestFlattenBoolean()
{
Query query = paW.Parse("A AND B OR C NOT (D AND E)");
FieldQuery fq = new FieldQuery(query, true, true);
HashSet<Query> flatQueries = new HashSet<Query>();
fq.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("A"), Tq("B"), Tq("C"));
}
public void testFlattenDisjunctionMaxQuery()
{
Query query = Dmq(Tq("A"), Tq("B"), PqF("C", "D"));
FieldQuery fq = new FieldQuery(query, true, true);
HashSet<Query> flatQueries = new HashSet<Query>();
fq.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("A"), Tq("B"), PqF("C", "D"));
}
public void TestFlattenTermAndPhrase()
{
Query query = paW.Parse("A AND \"B C\"");
FieldQuery fq = new FieldQuery(query, true, true);
HashSet<Query> flatQueries = new HashSet<Query>();
fq.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("A"), PqF("B", "C"));
}
public void TestFlattenTermAndPhrase2gram()
{
Query query = paB.Parse("AA AND BCD OR EFGH");
FieldQuery fq = new FieldQuery(query, true, true);
HashSet<Query> flatQueries = new HashSet<Query>();
fq.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("AA"), PqF("BC", "CD" ), PqF("EF", "FG", "GH"));
}
public void TestFlatten1TermPhrase()
{
Query query = PqF("A");
FieldQuery fq = new FieldQuery(query, true, true);
HashSet<Query> flatQueries = new HashSet<Query>();
fq.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("A"));
}
private FieldFragList Ffl(String queryValue, String indexValue)
{
Make1d1fIndex(indexValue);
Query query = paW.Parse(queryValue);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
return new SimpleFragListBuilder().CreateFieldFragList(fpl, 20);
}
public void TestFieldTermStackIndex1w2wSearch1term()
{
MakeIndex1w2w();
FieldQuery fq = new FieldQuery(Tq("pc"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(1, stack.termList.Count);
Assert.AreEqual("pc(3,5,1)", stack.Pop().ToString());
}
public void TestFieldTermStackIndex1w2wSearch1phrase()
{
MakeIndex1w2w();
FieldQuery fq = new FieldQuery(PqF("personal", "computer"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(2, stack.termList.Count);
Assert.AreEqual("personal(3,5,1)", stack.Pop().ToString());
Assert.AreEqual("computer(3,5,2)", stack.Pop().ToString());
}
public void Test2TermsIndex()
{
Make1d1fIndex("a a");
FieldQuery fq = new FieldQuery(Tq("a"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("a(1.0)((0,1))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("a(1.0)((2,3))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test1Phrase()
{
MakeIndex();
FieldQuery fq = new FieldQuery(PqF("c", "d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(3, stack.termList.Count);
Assert.AreEqual("c(10,11,5)", stack.Pop().ToString());
Assert.AreEqual("c(18,19,9)", stack.Pop().ToString());
Assert.AreEqual("d(20,21,10)", stack.Pop().ToString());
}
public void Test2TermsIndex()
{
Make1d1fIndex("a a");
FieldQuery fq = new FieldQuery(Tq("a"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("a(1.0)((0,1))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("a(1.0)((2,3))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test1PhraseLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(PqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("searchengines(1.0)((102,116))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("searchengines(1.0)((157,171))", fpl.phraseList.First.Next.Value.ToString());
}
public void TestCompleteSearchInLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(Preq("engin"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("engines(1.0)((109,116))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("engines(1.0)((164,171))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test1Phrase()
{
MakeIndex();
FieldQuery fq = new FieldQuery(PqF("c", "d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(3, stack.termList.Count);
Assert.AreEqual("c(10,11,5)", stack.Pop().ToString());
Assert.AreEqual("c(18,19,9)", stack.Pop().ToString());
Assert.AreEqual("d(20,21,10)", stack.Pop().ToString());
}
public void TestExpandNotFieldMatch()
{
Query dummy = PqF("DUMMY");
FieldQuery fq = new FieldQuery(dummy, true, false);
// f1:"a b",f2:"b c" => f1:"a b",f2:"b c",f1:"a b c"
HashSet <Query> flatQueries = new HashSet <Query>();
flatQueries.Add(Pq(F1, "a", "b"));
flatQueries.Add(Pq(F2, "b", "c"));
AssertCollectionQueries(fq.expand(flatQueries),
Pq(F1, "a", "b"), Pq(F2, "b", "c"), Pq(F1, "a", "b", "c"));
}
public void Test1PhraseB()
{
makeIndexB();
FieldQuery fq = new FieldQuery(PqF("ab", "bb"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(4, stack.termList.Count);
Assert.AreEqual("ab(2,4,2)", stack.Pop().ToString());
Assert.AreEqual("bb(3,5,3)", stack.Pop().ToString());
Assert.AreEqual("ab(6,8,6)", stack.Pop().ToString());
Assert.AreEqual("bb(7,9,7)", stack.Pop().ToString());
}
public void Test1PhraseShortMV()
{
MakeIndexShortMV();
FieldQuery fq = new FieldQuery(Tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
Assert.AreEqual(1, ffl.fragInfos.Count);
Assert.AreEqual("subInfos=(d((6,7)))/1.0(0,100)", ffl.fragInfos[0].ToString());
}
public void TestFieldPhraseListIndex2w1wSearch1partial()
{
MakeIndex2w1w();
FieldQuery fq = new FieldQuery(Tq("computer"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("computer(1.0)((3,20))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual(3, fpl.phraseList.First.Value.StartOffset);
Assert.AreEqual(20, fpl.phraseList.First.Value.EndOffset);
}
public void TestFlattenMultiPhraseQuery()
{
var query = new MultiPhraseQuery();
query.Add(new[] { new Term(F, "a1"), new Term(F, "a2") });
query.Add(new[] { new Term(F, "b1"), new Term(F, "b2") });
var fieldQuery = new FieldQuery(query, true, true);
var flatQueries = new HashSet <Query>();
fieldQuery.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("a1"), Tq("a2"), Tq("b1"), Tq("b2"));
}
public void Test1PhraseMVB()
{
MakeIndexLongMVB();
FieldQuery fq = new FieldQuery(PqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(4, stack.termList.Count);
Assert.AreEqual("sp(88,90,61)", stack.Pop().ToString());
Assert.AreEqual("pe(89,91,62)", stack.Pop().ToString());
Assert.AreEqual("ee(90,92,63)", stack.Pop().ToString());
Assert.AreEqual("ed(91,93,64)", stack.Pop().ToString());
}
public void TestFieldPhraseListIndex1w2wSearch1phrase()
{
MakeIndex1w2w();
FieldQuery fq = new FieldQuery(PqF("personal", "computer"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("personalcomputer(1.0)((3,5))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual(3, fpl.phraseList.First.Value.StartOffset);
Assert.AreEqual(5, fpl.phraseList.First.Value.EndOffset);
}
public void Test1PhraseLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(PqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
Assert.AreEqual(1, ffl.fragInfos.Count);
Assert.AreEqual("subInfos=(searchengines((102,116))searchengines((157,171)))/2.0(96,196)", ffl.fragInfos[0].ToString());
}
public void Test1PhraseLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(PqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(4, stack.termList.Count);
Assert.AreEqual("search(102,108,14)", stack.Pop().ToString());
Assert.AreEqual("engines(109,116,15)", stack.Pop().ToString());
Assert.AreEqual("search(157,163,24)", stack.Pop().ToString());
Assert.AreEqual("engines(164,171,25)", stack.Pop().ToString());
}
public void Test1PhraseLongMVB()
{
MakeIndexLongMVB();
FieldQuery fq = new FieldQuery(PqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
Assert.AreEqual(1, ffl.fragInfos.Count);
Assert.AreEqual("subInfos=(sppeeeed((88,93)))/1.0(82,182)", ffl.fragInfos[0].ToString());
}
public void TestPhraseSlop()
{
Make1d1fIndex("c a a b c");
FieldQuery fq = new FieldQuery(pqF(2F, 1, "a", "c"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("ac(2.0)((4,5)(8,9))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual(4, fpl.phraseList.First.Value.StartOffset);
Assert.AreEqual(9, fpl.phraseList.First.Value.EndOffset);
}
public void Test1PhraseShortMV()
{
MakeIndexShortMV();
FieldQuery fq = new FieldQuery(Tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
Assert.AreEqual("a b c <b>d</b> e", sfb.CreateFragment(reader, 0, F, ffl));
}
public void TestUnstoredField()
{
MakeUnstoredIndex();
FieldQuery fq = new FieldQuery(Tq("aaa"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
Assert.IsNull(sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1PhraseLongMVB()
{
MakeIndexLongMVB();
FieldQuery fq = new FieldQuery(PqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
Assert.AreEqual("ssing <b>speed</b>, the", sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1Term()
{
MakeIndex();
FieldQuery fq = new FieldQuery(Tq("a"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(6, stack.termList.Count);
Assert.AreEqual("a(0,1,0)", stack.Pop().ToString());
Assert.AreEqual("a(2,3,1)", stack.Pop().ToString());
Assert.AreEqual("a(4,5,2)", stack.Pop().ToString());
Assert.AreEqual("a(12,13,6)", stack.Pop().ToString());
Assert.AreEqual("a(28,29,14)", stack.Pop().ToString());
Assert.AreEqual("a(32,33,16)", stack.Pop().ToString());
}
public void Test1PhraseLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(PqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
Assert.AreEqual(" most <b>search engines</b> use only one of these methods. Even the <b>search engines</b> that says they can use t",
sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1PhraseIndexB()
{
// 01 12 23 34 45 56 67 78 (offsets)
// bb|bb|ba|ac|cb|ba|ab|bc
// 0 1 2 3 4 5 6 7 (positions)
Make1d1fIndexB("bbbacbabc");
FieldQuery fq = new FieldQuery(PqF("ba", "ac"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("baac(1.0)((2,5))", fpl.phraseList.First.Value.ToString());
}
public void Test2ConcatTermsIndexB()
{
// 01 12 23 (offsets)
// ab|ba|ab
// 0 1 2 (positions)
Make1d1fIndexB("abab");
FieldQuery fq = new FieldQuery(Tq("ab"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((0,2))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("ab(1.0)((2,4))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test2PhrasesOverlap()
{
Make1d1fIndex("d a b c d");
BooleanQuery query = new BooleanQuery();
query.Add(PqF("a", "b"), Occur.SHOULD);
query.Add(PqF("b", "c"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("abc(1.0)((2,7))", fpl.phraseList.First.Value.ToString());
}
public void Test1Term()
{
MakeIndex();
FieldQuery fq = new FieldQuery(Tq("a"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(6, stack.termList.Count);
Assert.AreEqual("a(0,1,0)", stack.Pop().ToString());
Assert.AreEqual("a(2,3,1)", stack.Pop().ToString());
Assert.AreEqual("a(4,5,2)", stack.Pop().ToString());
Assert.AreEqual("a(12,13,6)", stack.Pop().ToString());
Assert.AreEqual("a(28,29,14)", stack.Pop().ToString());
Assert.AreEqual("a(32,33,16)", stack.Pop().ToString());
}
/// <summary>
/// a constructor.
/// </summary>
/// <param name="reader">IndexReader of the index</param>
/// <param name="docId">document id to be highlighted</param>
/// <param name="fieldName">field of the document to be highlighted</param>
/// <param name="fieldQuery">FieldQuery object</param>
#if LUCENENET_350 //Lucene.Net specific code. See https://issues.apache.org/jira/browse/LUCENENET-350
public FieldTermStack(IndexReader reader, int docId, String fieldName, FieldQuery fieldQuery, IState state)
{
this.fieldName = fieldName;
List <string> termSet = fieldQuery.getTermSet(fieldName);
// just return to make null snippet if un-matched fieldName specified when fieldMatch == true
if (termSet == null)
{
return;
}
//TermFreqVector tfv = reader.GetTermFreqVector(docId, fieldName);
VectorHighlightMapper tfv = new VectorHighlightMapper(termSet);
reader.GetTermFreqVector(docId, fieldName, tfv, state);
if (tfv.Size == 0)
{
return; // just return to make null snippets
}
string[] terms = tfv.GetTerms();
foreach (String term in terms)
{
if (!StringUtils.AnyTermMatch(termSet, term))
{
continue;
}
int index = tfv.IndexOf(term);
TermVectorOffsetInfo[] tvois = tfv.GetOffsets(index);
if (tvois == null)
{
return; // just return to make null snippets
}
int[] poss = tfv.GetTermPositions(index);
if (poss == null)
{
return; // just return to make null snippets
}
for (int i = 0; i < tvois.Length; i++)
{
termList.AddLast(new TermInfo(term, tvois[i].StartOffset, tvois[i].EndOffset, poss[i]));
}
}
// sort by position
//Collections.sort(termList);
Sort(termList);
}
public void TestNoExpand()
{
Query dummy = PqF("DUMMY");
FieldQuery fq = new FieldQuery(dummy, true, true);
// "a b","c d" => "a b","c d"
HashSet <Query> flatQueries = new HashSet <Query>();
flatQueries.Add(PqF("a", "b"));
flatQueries.Add(PqF("c", "d"));
AssertCollectionQueries(fq.expand(flatQueries),
PqF("a", "b"), PqF("c", "d"));
// "a","a b" => "a", "a b"
flatQueries = new HashSet <Query>();
flatQueries.Add(Tq("a"));
flatQueries.Add(PqF("a", "b"));
AssertCollectionQueries(fq.expand(flatQueries),
Tq("a"), PqF("a", "b"));
// "a b","b" => "a b", "b"
flatQueries = new HashSet <Query>();
flatQueries.Add(PqF("a", "b"));
flatQueries.Add(Tq("b"));
AssertCollectionQueries(fq.expand(flatQueries),
PqF("a", "b"), Tq("b"));
// "a b c","b c" => "a b c","b c"
flatQueries = new HashSet <Query>();
flatQueries.Add(PqF("a", "b", "c"));
flatQueries.Add(PqF("b", "c"));
AssertCollectionQueries(fq.expand(flatQueries),
PqF("a", "b", "c"), PqF("b", "c"));
// "a b","a b c" => "a b","a b c"
flatQueries = new HashSet <Query>();
flatQueries.Add(PqF("a", "b"));
flatQueries.Add(PqF("a", "b", "c"));
AssertCollectionQueries(fq.expand(flatQueries),
PqF("a", "b"), PqF("a", "b", "c"));
// "a b c","b d e" => "a b c","b d e"
flatQueries = new HashSet <Query>();
flatQueries.Add(PqF("a", "b", "c"));
flatQueries.Add(PqF("b", "d", "e"));
AssertCollectionQueries(fq.expand(flatQueries),
PqF("a", "b", "c"), PqF("b", "d", "e"));
}
public void Test2TermsB()
{
makeIndexB();
BooleanQuery query = new BooleanQuery();
query.Add(Tq("bc"), Occur.SHOULD);
query.Add(Tq("ef"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(3, stack.termList.Count);
Assert.AreEqual("bc(4,6,4)", stack.Pop().ToString());
Assert.AreEqual("bc(8,10,8)", stack.Pop().ToString());
Assert.AreEqual("ef(11,13,11)", stack.Pop().ToString());
}
public void TestSearchLongestPhrase()
{
Make1d1fIndex("d a b d c a b c");
BooleanQuery query = new BooleanQuery();
query.Add(PqF("a", "b"), Occur.SHOULD);
query.Add(PqF("a", "b", "c"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((2,5))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("abc(1.0)((10,15))", fpl.phraseList.First.Next.Value.ToString());
}
public void TestFlattenSpanQuery()
{
var clauses = new SpanQuery[]
{
new SpanTermQuery(new Term(F, "a")),
new SpanTermQuery(new Term(F, "b")),
new SpanTermQuery(new Term(F, "c")),
};
var query = new SpanNearQuery(clauses, 3, true);
var fieldQuery = new FieldQuery(query, true, true);
var flatQueries = new HashSet <Query>();
fieldQuery.flatten(query, flatQueries);
AssertCollectionQueries(flatQueries, Tq("a"), Tq("b"), Tq("c"));
}
public void Test1PhraseIndex()
{
Make1d1fIndex("a b");
FieldQuery fq = new FieldQuery(PqF("a", "b"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((0,3))", fpl.phraseList.First.Value.ToString());
fq = new FieldQuery(Tq("b"), true, true);
stack = new FieldTermStack(reader, 0, F, fq);
fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("b(1.0)((2,3))", fpl.phraseList.First.Value.ToString());
}
public void TestGetRootMapNotFieldMatch()
{
Query dummy = PqF("DUMMY");
FieldQuery fq = new FieldQuery(dummy, true, false);
QueryPhraseMap rootMap1 = fq.getRootMap(Tq("a"));
QueryPhraseMap rootMap2 = fq.getRootMap(Tq("a"));
Assert.IsTrue(rootMap1 == rootMap2);
QueryPhraseMap rootMap3 = fq.getRootMap(Tq("b"));
Assert.IsTrue(rootMap1 == rootMap3);
QueryPhraseMap rootMap4 = fq.getRootMap(Tq(F1, "b"));
Assert.IsTrue(rootMap4 == rootMap3);
}
public void TestFieldTermStackIndex1wSearch2terms()
{
MakeIndex1w();
BooleanQuery bq = new BooleanQuery();
bq.Add(Tq("Mac"), Occur.SHOULD);
bq.Add(Tq("MacBook"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(bq, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(2, stack.termList.Count);
Dictionary<String, String> expectedSet = new Dictionary<String, String>();
expectedSet.Add("Mac(11,20,3)","");
expectedSet.Add("MacBook(11,20,3)","");
Assert.IsTrue(expectedSet.ContainsKey(stack.Pop().ToString()));
Assert.IsTrue(expectedSet.ContainsKey(stack.Pop().ToString()));
}
public void TestGetFieldTermMap()
{
Query query = Tq("a");
FieldQuery fq = new FieldQuery(query, true, true);
QueryPhraseMap pqm = fq.GetFieldTermMap(F, "a");
Assert.NotNull(pqm);
Assert.IsTrue(pqm.IsTerminal);
pqm = fq.GetFieldTermMap(F, "b");
Assert.Null(pqm);
pqm = fq.GetFieldTermMap(F1, "a");
Assert.Null(pqm);
}
public void Test2Terms()
{
MakeIndex();
BooleanQuery query = new BooleanQuery();
query.Add(Tq("b"), Occur.SHOULD);
query.Add(Tq("c"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
Assert.AreEqual(8, stack.termList.Count);
Assert.AreEqual("b(6,7,3)", stack.Pop().ToString());
Assert.AreEqual("b(8,9,4)", stack.Pop().ToString());
Assert.AreEqual("c(10,11,5)", stack.Pop().ToString());
Assert.AreEqual("b(14,15,7)", stack.Pop().ToString());
Assert.AreEqual("b(16,17,8)", stack.Pop().ToString());
Assert.AreEqual("c(18,19,9)", stack.Pop().ToString());
Assert.AreEqual("b(26,27,13)", stack.Pop().ToString());
Assert.AreEqual("b(30,31,15)", stack.Pop().ToString());
}
public void Test1PhraseLongMVB()
{
MakeIndexLongMVB();
FieldQuery fq = new FieldQuery(PqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("sppeeeed(1.0)((88,93))", fpl.phraseList.First.Value.ToString());
}
public void Test1PhraseLongMV()
{
MakeIndexLongMV();
FieldQuery fq = new FieldQuery(PqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("searchengines(1.0)((102,116))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("searchengines(1.0)((157,171))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test1PhraseShortMV()
{
MakeIndexShortMV();
FieldQuery fq = new FieldQuery(Tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("d(1.0)((6,7))", fpl.phraseList.First.Value.ToString());
}
public void TestSearchLongestPhrase()
{
Make1d1fIndex("d a b d c a b c");
BooleanQuery query = new BooleanQuery();
query.Add(PqF("a", "b"), Lucene.Net.Search.BooleanClause.Occur.SHOULD);
query.Add(PqF("a", "b", "c"), Lucene.Net.Search.BooleanClause.Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((2,5))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("abc(1.0)((10,15))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test3TermsPhrase()
{
Make1d1fIndex("d a b a b c d");
FieldQuery fq = new FieldQuery(PqF("a", "b", "c"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("abc(1.0)((6,11))", fpl.phraseList.First.Value.ToString());
}
public void TestPhraseSlop()
{
Make1d1fIndex("c a a b c");
FieldQuery fq = new FieldQuery(pqF(2F, 1, "a", "c"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("ac(2.0)((4,5)(8,9))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual(4, fpl.phraseList.First.Value.GetStartOffset());
Assert.AreEqual(9, fpl.phraseList.First.Value.GetEndOffset());
}
private string GetHighlightedText(FastVectorHighlighter highlighter, FieldQuery fieldQuery, IndexSearcher searcher, ScoreDoc match, string tag, int length)
{
var s = highlighter.GetBestFragment(fieldQuery, searcher.IndexReader, match.Doc, tag, length);
if (!string.IsNullOrEmpty(s))
{
s = HttpUtility.HtmlEncode(s).Replace(HighlightPreTag, HtmlPreTag).Replace(HighlightPostTag, HtmlPostTag);
}
return s;
}
public void Test1PhraseIndexB()
{
// 01 12 23 34 45 56 67 78 (offsets)
// bb|bb|ba|ac|cb|ba|ab|bc
// 0 1 2 3 4 5 6 7 (positions)
Make1d1fIndexB("bbbacbabc");
FieldQuery fq = new FieldQuery(PqF("ba", "ac"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("baac(1.0)((2,5))", fpl.phraseList.First.Value.ToString());
}
public QueryPhraseMap(FieldQuery fieldQuery)
{
this.fieldQuery = fieldQuery;
}
private FieldFragList GetFieldFragList(FieldQuery fieldQuery, IndexReader reader, int docId,
String fieldName, int fragCharSize)
{
FieldTermStack fieldTermStack = new FieldTermStack(reader, docId, fieldName, fieldQuery);
FieldPhraseList fieldPhraseList = new FieldPhraseList(fieldTermStack, fieldQuery, phraseLimit);
return fragListBuilder.CreateFieldFragList(fieldPhraseList, fragCharSize);
}
private void SetupHighlighter(Query luceneQuery)
{
if (indexQuery.HighlightedFields != null && indexQuery.HighlightedFields.Length > 0)
{
highlighter = new FastVectorHighlighter(
FastVectorHighlighter.DEFAULT_PHRASE_HIGHLIGHT,
FastVectorHighlighter.DEFAULT_FIELD_MATCH,
new SimpleFragListBuilder(),
new SimpleFragmentsBuilder(
indexQuery.HighlighterPreTags != null && indexQuery.HighlighterPreTags.Any()
? indexQuery.HighlighterPreTags
: BaseFragmentsBuilder.COLORED_PRE_TAGS,
indexQuery.HighlighterPostTags != null && indexQuery.HighlighterPostTags.Any()
? indexQuery.HighlighterPostTags
: BaseFragmentsBuilder.COLORED_POST_TAGS));
fieldQuery = highlighter.GetFieldQuery(luceneQuery);
}
}
public void Test2PhrasesOverlap()
{
Make1d1fIndex("d a b c d");
BooleanQuery query = new BooleanQuery();
query.Add(PqF("a", "b"), Occur.SHOULD);
query.Add(PqF("b", "c"), Occur.SHOULD);
FieldQuery fq = new FieldQuery(query, true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("abc(1.0)((2,7))", fpl.phraseList.First.Value.ToString());
}
public void Test2Terms1PhraseIndex()
{
Make1d1fIndex("c a a b");
// phraseHighlight = true
FieldQuery fq = new FieldQuery(PqF("a", "b"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(1, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((4,7))", fpl.phraseList.First.Value.ToString());
// phraseHighlight = false
fq = new FieldQuery(PqF("a", "b"), false, true);
stack = new FieldTermStack(reader, 0, F, fq);
fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("a(1.0)((2,3))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("ab(1.0)((4,7))", fpl.phraseList.First.Next.Value.ToString());
}
public void Test2ConcatTermsIndexB()
{
// 01 12 23 (offsets)
// ab|ba|ab
// 0 1 2 (positions)
Make1d1fIndexB("abab");
FieldQuery fq = new FieldQuery(Tq("ab"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
Assert.AreEqual(2, fpl.phraseList.Count);
Assert.AreEqual("ab(1.0)((0,2))", fpl.phraseList.First.Value.ToString());
Assert.AreEqual("ab(1.0)((2,4))", fpl.phraseList.First.Next.Value.ToString());
}
/// <summary>
/// return the best fragment.
/// </summary>
/// <param name="fieldQuery">FieldQuery object</param>
/// <param name="reader">IndexReader of the index</param>
/// <param name="docId">document id to be highlighted</param>
/// <param name="fieldName">field of the document to be highlighted</param>
/// <param name="fragCharSize">the length (number of chars) of a fragment</param>
/// <returns>the best fragment (snippet) string</returns>
public String GetBestFragment(FieldQuery fieldQuery, IndexReader reader, int docId,
String fieldName, int fragCharSize)
{
FieldFragList fieldFragList = GetFieldFragList(fieldQuery, reader, docId, fieldName, fragCharSize);
return fragmentsBuilder.CreateFragment(reader, docId, fieldName, fieldFragList);
}