public void TestDiscreteMultiValueHighlighting()
{
makeIndexShortMV();
FieldQuery fq = new FieldQuery(tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.IsDiscreteMultiValueHighlighting = (true);
assertEquals("<b>d</b> e", sfb.CreateFragment(reader, 0, F, ffl));
make1dmfIndex("some text to highlight", "highlight other text");
fq = new FieldQuery(tq("text"), true, true);
stack = new FieldTermStack(reader, 0, F, fq);
fpl = new FieldPhraseList(stack, fq);
sflb = new SimpleFragListBuilder();
ffl = sflb.CreateFieldFragList(fpl, 32);
String[] result = sfb.CreateFragments(reader, 0, F, ffl, 3);
assertEquals(2, result.Length);
assertEquals("some <b>text</b> to highlight", result[0]);
assertEquals("highlight other <b>text</b>", result[1]);
fq = new FieldQuery(tq("highlight"), true, true);
stack = new FieldTermStack(reader, 0, F, fq);
fpl = new FieldPhraseList(stack, fq);
sflb = new SimpleFragListBuilder();
ffl = sflb.CreateFieldFragList(fpl, 32);
result = sfb.CreateFragments(reader, 0, F, ffl, 3);
assertEquals(2, result.Length);
assertEquals("text to <b>highlight</b>", result[0]);
assertEquals("<b>highlight</b> other text", result[1]);
}
public void TestTagsAndEncoder()
{
FieldFragList ffl = Ffl(new TermQuery(new Term(F, "a")), "<h1> a </h1>");
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" };
String[] postTags = { "]" };
assertEquals("<h1> [a] </h1>",
sfb.CreateFragment(reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder()));
}
public void Test2Frags()
{
FieldFragList ffl = Ffl(new TermQuery(new Term(F, "a")), "a b b b b b b b b b b b a b a b");
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] f = sfb.CreateFragments(reader, 0, F, ffl, 3);
// 3 snippets requested, but should be 2
assertEquals(2, f.Length);
assertEquals("<b>a</b> b b b b b b b b b b", f[0]);
assertEquals("b b <b>a</b> b <b>a</b> b", f[1]);
}
public void Test1TermIndex()
{
FieldFragList ffl = Ffl(new TermQuery(new Term(F, "a")), "a");
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals("<b>a</b>", sfb.CreateFragment(reader, 0, F, ffl));
// change tags
sfb = new SimpleFragmentsBuilder(new String[] { "[" }, new String[] { "]" });
assertEquals("[a]", sfb.CreateFragment(reader, 0, F, ffl));
}
public void TestUnstoredField()
{
makeUnstoredIndex();
FieldQuery fq = new FieldQuery(tq("aaa"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertNull(sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1PhraseLongMVB()
{
makeIndexLongMVB();
FieldQuery fq = new FieldQuery(pqF("sp", "pe", "ee", "ed"), true, true); // "speed" -(2gram)-> "sp","pe","ee","ed"
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals("additional hardware. \nWhen you talk about processing <b>speed</b>, the", sfb.CreateFragment(reader, 0, F, ffl));
}
public void TestMVSeparator()
{
makeIndexShortMV();
FieldQuery fq = new FieldQuery(tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.MultiValuedSeparator = ('/');
assertEquals("//a b c//<b>d</b> e", sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1PhraseLongMV()
{
makeIndexLongMV();
FieldQuery fq = new FieldQuery(pqF("search", "engines"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
assertEquals("customization: The most <b>search engines</b> use only one of these methods. Even the <b>search engines</b> that says they can",
sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test1PhraseShortMV()
{
makeIndexShortMV();
FieldQuery fq = new FieldQuery(tq("d"), true, true);
FieldTermStack stack = new FieldTermStack(reader, 0, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 100);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
// Should we probably be trimming?
assertEquals(" a b c <b>d</b> e", sfb.CreateFragment(reader, 0, F, ffl));
}
public void Test3Frags()
{
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.Add(new TermQuery(new Term(F, "a")), Occur.SHOULD);
booleanQuery.Add(new TermQuery(new Term(F, "c")), Occur.SHOULD);
FieldFragList ffl = Ffl(booleanQuery, "a b b b b b b b b b b b a b a b b b b b c a a b b");
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] f = sfb.CreateFragments(reader, 0, F, ffl, 3);
assertEquals(3, f.Length);
assertEquals("<b>a</b> b b b b b b b b b b", f[0]);
assertEquals("b b <b>a</b> b <b>a</b> b b b b b c", f[1]);
assertEquals("<b>c</b> <b>a</b> <b>a</b> b b", f[2]);
}
public void TestRandomDiscreteMultiValueHighlighting()
{
String[] randomValues = new String[3 + Random.nextInt(10 * RANDOM_MULTIPLIER)];
for (int i = 0; i < randomValues.Length; i++)
{
String randomValue;
do
{
randomValue = TestUtil.RandomSimpleString(Random);
} while ("".Equals(randomValue, StringComparison.Ordinal));
randomValues[i] = randomValue;
}
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
Random,
dir,
NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random)).SetMergePolicy(NewLogMergePolicy()));
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = (true);
customType.StoreTermVectorOffsets = (true);
customType.StoreTermVectorPositions = (true);
int numDocs = randomValues.Length * 5;
int numFields = 2 + Random.nextInt(5);
int numTerms = 2 + Random.nextInt(3);
List <Doc> docs = new List <Doc>(numDocs);
List <Document> documents = new List <Document>(numDocs);
IDictionary <String, ISet <int> > valueToDocId = new HashMap <String, ISet <int> >();
for (int i = 0; i < numDocs; i++)
{
Document document = new Document();
String[][] fields = RectangularArrays.ReturnRectangularArray <string>(numFields, numTerms); //new String[numFields][numTerms];
for (int j = 0; j < numFields; j++)
{
String[] fieldValues = new String[numTerms];
fieldValues[0] = getRandomValue(randomValues, valueToDocId, i);
StringBuilder builder = new StringBuilder(fieldValues[0]);
for (int k = 1; k < numTerms; k++)
{
fieldValues[k] = getRandomValue(randomValues, valueToDocId, i);
builder.Append(' ').Append(fieldValues[k]);
}
document.Add(new Field(F, builder.ToString(), customType));
fields[j] = fieldValues;
}
docs.Add(new Doc(fields));
documents.Add(document);
}
writer.AddDocuments(documents);
writer.Dispose();
IndexReader reader = DirectoryReader.Open(dir);
try
{
int highlightIters = 1 + Random.nextInt(120 * RANDOM_MULTIPLIER);
for (int highlightIter = 0; highlightIter < highlightIters; highlightIter++)
{
Console.WriteLine($"Highlighter iter: {highlightIter}");
String queryTerm = randomValues[Random.nextInt(randomValues.Length)];
int randomHit = valueToDocId[queryTerm].First();
List <StringBuilder> builders = new List <StringBuilder>();
foreach (String[] fieldValues in docs[randomHit].fieldValues)
{
StringBuilder builder = new StringBuilder();
bool hit = false;
for (int i = 0; i < fieldValues.Length; i++)
{
if (queryTerm.Equals(fieldValues[i], StringComparison.Ordinal))
{
builder.Append("<b>").Append(queryTerm).Append("</b>");
hit = true;
}
else
{
builder.Append(fieldValues[i]);
}
if (i != fieldValues.Length - 1)
{
builder.Append(' ');
}
}
if (hit)
{
builders.Add(builder);
}
}
FieldQuery fq = new FieldQuery(tq(queryTerm), true, true);
FieldTermStack stack = new FieldTermStack(reader, randomHit, F, fq);
FieldPhraseList fpl = new FieldPhraseList(stack, fq);
SimpleFragListBuilder sflb = new SimpleFragListBuilder(100);
FieldFragList ffl = sflb.CreateFieldFragList(fpl, 300);
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.IsDiscreteMultiValueHighlighting = (true);
String[] actualFragments = sfb.CreateFragments(reader, randomHit, F, ffl, numFields);
assertEquals(builders.Count, actualFragments.Length);
for (int i = 0; i < actualFragments.Length; i++)
{
assertEquals(builders[i].ToString(), actualFragments[i]);
}
}
}
finally
{
reader.Dispose();
dir.Dispose();
}
}