public static void processFragment(ref TextFragment frag)
{
if (frag.GetScore() > 0 && !frag.Visited)
{
frag.MarkedUpText.Append("!!!!");
frag.Visited = true;
}
}
/// <seealso cref="IScorer.StartFragment"/>
public virtual void StartFragment(TextFragment newFragment)
{
foundTerms = new JCG.HashSet <string>();
totalScore = 0;
}
/// <summary>
/// Improves readability of a score-sorted list of TextFragments by merging any fragments
/// that were contiguous in the original text into one larger fragment with the correct order.
/// This will leave a "null" in the array entry for the lesser scored fragment.
/// </summary>
/// <param name="frag">An array of document fragments in descending score</param>
private static void MergeContiguousFragments(TextFragment[] frag) // LUCENENET: CA1822: Mark members as static
{
bool mergingStillBeingDone;
if (frag.Length > 1)
{
do
{
mergingStillBeingDone = false; //initialise loop control flag
//for each fragment, scan other frags looking for contiguous blocks
for (int i = 0; i < frag.Length; i++)
{
if (frag[i] == null)
{
continue;
}
//merge any contiguous blocks
for (int x = 0; x < frag.Length; x++)
{
if (frag[x] == null)
{
continue;
}
if (frag[i] == null)
{
break;
}
TextFragment frag1 = null;
TextFragment frag2 = null;
int frag1Num = 0;
int frag2Num = 0;
int bestScoringFragNum;
int worstScoringFragNum;
//if blocks are contiguous....
if (frag[i].Follows(frag[x]))
{
frag1 = frag[x];
frag1Num = x;
frag2 = frag[i];
frag2Num = i;
}
else if (frag[x].Follows(frag[i]))
{
frag1 = frag[i];
frag1Num = i;
frag2 = frag[x];
frag2Num = x;
}
//merging required..
if (frag1 != null)
{
if (frag1.Score > frag2.Score)
{
bestScoringFragNum = frag1Num;
worstScoringFragNum = frag2Num;
}
else
{
bestScoringFragNum = frag2Num;
worstScoringFragNum = frag1Num;
}
frag1.Merge(frag2);
frag[worstScoringFragNum] = null;
mergingStillBeingDone = true;
frag[bestScoringFragNum] = frag1;
}
}
}
} while (mergingStillBeingDone);
}
}
/// <summary>
/// Low level api to get the most relevant (formatted) sections of the document.
/// This method has been made public to allow visibility of score information held in <see cref="TextFragment"/> objects.
/// Thanks to Jason Calabrese for help in redefining the interface.
/// </summary>
/// <exception cref="IOException">If there is a low-level I/O error</exception>
/// <exception cref="InvalidTokenOffsetsException">thrown if any token's EndOffset exceeds the provided text's length</exception>
public TextFragment[] GetBestTextFragments(
TokenStream tokenStream,
string text,
bool mergeContiguousFragments,
int maxNumFragments)
{
var docFrags = new JCG.List <TextFragment>();
var newText = new StringBuilder();
var termAtt = tokenStream.AddAttribute <ICharTermAttribute>();
var offsetAtt = tokenStream.AddAttribute <IOffsetAttribute>();
tokenStream.Reset();
var currentFrag = new TextFragment(newText, newText.Length, docFrags.Count);
if (_fragmentScorer is QueryScorer queryScorer)
{
queryScorer.SetMaxDocCharsToAnalyze(_maxDocCharsToAnalyze);
}
var newStream = _fragmentScorer.Init(tokenStream);
if (newStream != null)
{
tokenStream = newStream;
}
_fragmentScorer.StartFragment(currentFrag);
docFrags.Add(currentFrag);
var fragQueue = new FragmentQueue(maxNumFragments);
try
{
string tokenText;
int startOffset;
int endOffset;
int lastEndOffset = 0;
_textFragmenter.Start(text, tokenStream);
var tokenGroup = new TokenGroup(tokenStream);
for (bool next = tokenStream.IncrementToken();
next && (offsetAtt.StartOffset < _maxDocCharsToAnalyze);
next = tokenStream.IncrementToken())
{
if ((offsetAtt.EndOffset > text.Length)
||
(offsetAtt.StartOffset > text.Length)
)
{
throw new InvalidTokenOffsetsException("Token " + termAtt.ToString()
+ " exceeds length of provided text sized " + text.Length);
}
if ((tokenGroup.NumTokens > 0) && (tokenGroup.IsDistinct()))
{
//the current token is distinct from previous tokens -
// markup the cached token group info
startOffset = tokenGroup.MatchStartOffset;
endOffset = tokenGroup.MatchEndOffset;
tokenText = text.Substring(startOffset, endOffset - startOffset);
string markedUpText = _formatter.HighlightTerm(_encoder.EncodeText(tokenText), tokenGroup);
//store any whitespace etc from between this and last group
if (startOffset > lastEndOffset)
{
newText.Append(_encoder.EncodeText(text.Substring(lastEndOffset, startOffset - lastEndOffset)));
}
newText.Append(markedUpText);
lastEndOffset = Math.Max(endOffset, lastEndOffset);
tokenGroup.Clear();
//check if current token marks the start of a new fragment
if (_textFragmenter.IsNewFragment())
{
currentFrag.Score = _fragmentScorer.FragmentScore;
//record stats for a new fragment
currentFrag.TextEndPos = newText.Length;
currentFrag = new TextFragment(newText, newText.Length, docFrags.Count);
_fragmentScorer.StartFragment(currentFrag);
docFrags.Add(currentFrag);
}
}
tokenGroup.AddToken(_fragmentScorer.GetTokenScore());
// if(lastEndOffset>maxDocBytesToAnalyze)
// {
// break;
// }
}
currentFrag.Score = _fragmentScorer.FragmentScore;
if (tokenGroup.NumTokens > 0)
{
//flush the accumulated text (same code as in above loop)
startOffset = tokenGroup.MatchStartOffset;
endOffset = tokenGroup.MatchEndOffset;
tokenText = text.Substring(startOffset, endOffset - startOffset);
var markedUpText = _formatter.HighlightTerm(_encoder.EncodeText(tokenText), tokenGroup);
//store any whitespace etc from between this and last group
if (startOffset > lastEndOffset)
{
newText.Append(_encoder.EncodeText(text.Substring(lastEndOffset, startOffset - lastEndOffset)));
}
newText.Append(markedUpText);
lastEndOffset = Math.Max(lastEndOffset, endOffset);
}
//Test what remains of the original text beyond the point where we stopped analyzing
if (
// if there is text beyond the last token considered..
(lastEndOffset < text.Length)
&&
// and that text is not too large...
(text.Length <= _maxDocCharsToAnalyze)
)
{
//append it to the last fragment
newText.Append(_encoder.EncodeText(text.Substring(lastEndOffset)));
}
currentFrag.TextEndPos = newText.Length;
//sort the most relevant sections of the text
foreach (var f in docFrags)
{
currentFrag = f;
//If you are running with a version of Lucene before 11th Sept 03
// you do not have PriorityQueue.insert() - so uncomment the code below
/*
* if (currentFrag.getScore() >= minScore)
* {
* fragQueue.put(currentFrag);
* if (fragQueue.size() > maxNumFragments)
* { // if hit queue overfull
* fragQueue.pop(); // remove lowest in hit queue
* minScore = ((TextFragment) fragQueue.top()).getScore(); // reset minScore
* }
* }
*/
//The above code caused a problem as a result of Christoph Goller's 11th Sept 03
//fix to PriorityQueue. The correct method to use here is the new "insert" method
// USE ABOVE CODE IF THIS DOES NOT COMPILE!
fragQueue.InsertWithOverflow(currentFrag);
}
//return the most relevant fragments
var frag = new TextFragment[fragQueue.Count];
for (int i = frag.Length - 1; i >= 0; i--)
{
frag[i] = fragQueue.Pop();
}
//merge any contiguous fragments to improve readability
if (mergeContiguousFragments)
{
MergeContiguousFragments(frag);
JCG.List <TextFragment> fragTexts = new JCG.List <TextFragment>();
for (int i = 0; i < frag.Length; i++)
{
if ((frag[i] != null) && (frag[i].Score > 0))
{
fragTexts.Add(frag[i]);
}
}
frag = new TextFragment[fragTexts.Count];
fragTexts.CopyTo(frag);
}
return(frag);
}
finally
{
if (tokenStream != null)
{
try
{
tokenStream.End();
tokenStream.Dispose();
}
catch (Exception e) when(e.IsException())
{
}
}
}
}
/// <summary>
/// true if this fragment follows the one passed
/// </summary>
public virtual bool Follows(TextFragment fragment)
{
return(TextStartPos == fragment.TextEndPos);
}
/// <param name="frag2">Fragment to be merged into this one</param>
public virtual void Merge(TextFragment frag2)
{
TextEndPos = frag2.TextEndPos;
Score = Math.Max(Score, frag2.Score);
}
/// <seealso cref="IScorer.StartFragment"/>
public void StartFragment(TextFragment newFragment)
{
foundTerms = Support.Compatibility.SetFactory.GetSet<string>();
totalScore = 0;
}
/// <seealso cref="IScorer.StartFragment"/>
public virtual void StartFragment(TextFragment newFragment)
{
foundTerms = Support.Compatibility.SetFactory.CreateHashSet <string>();
totalScore = 0;
}
/// <param name="frag2">
/// Fragment to be merged into this one
/// </param>
public virtual void Merge(TextFragment frag2)
{
textEndPos = frag2.textEndPos;
score = System.Math.Max(score, frag2.score);
}
/// <param name="fragment">
/// </param>
/// <returns> true if this fragment follows the one passed
/// </returns>
public virtual bool Follows(TextFragment fragment)
{
return textStartPos == fragment.textEndPos;
}
/// <summary>Improves readability of a score-sorted list of TextFragments by merging any fragments
/// that were contiguous in the original text into one larger fragment with the correct order.
/// This will leave a "null" in the array entry for the lesser scored fragment.
///
/// </summary>
/// <param name="frag">An array of document fragments in descending score
/// </param>
private void MergeContiguousFragments(TextFragment[] frag)
{
bool mergingStillBeingDone;
if (frag.Length > 1)
do
{
mergingStillBeingDone = false; //initialise loop control flag
//for each fragment, scan other frags looking for contiguous blocks
for (int i = 0; i < frag.Length; i++)
{
if (frag[i] == null)
{
continue;
}
//merge any contiguous blocks
for (int x = 0; x < frag.Length; x++)
{
if (frag[x] == null)
{
continue;
}
if (frag[i] == null)
{
break;
}
TextFragment frag1 = null;
TextFragment frag2 = null;
int frag1Num = 0;
int frag2Num = 0;
int bestScoringFragNum;
int worstScoringFragNum;
//if blocks are contiguous....
if (frag[i].Follows(frag[x]))
{
frag1 = frag[x];
frag1Num = x;
frag2 = frag[i];
frag2Num = i;
}
else if (frag[x].Follows(frag[i]))
{
frag1 = frag[i];
frag1Num = i;
frag2 = frag[x];
frag2Num = x;
}
//merging required..
if (frag1 != null)
{
if (frag1.GetScore() > frag2.GetScore())
{
bestScoringFragNum = frag1Num;
worstScoringFragNum = frag2Num;
}
else
{
bestScoringFragNum = frag2Num;
worstScoringFragNum = frag1Num;
}
frag1.Merge(frag2);
frag[worstScoringFragNum] = null;
mergingStillBeingDone = true;
frag[bestScoringFragNum] = frag1;
}
}
}
}
while (mergingStillBeingDone);
}
/// <summary> Low level api to get the most relevant (formatted) sections of the document.
/// This method has been made public to allow visibility of score information held in TextFragment objects.
/// Thanks to Jason Calabrese for help in redefining the interface.
/// </summary>
/// <param name="">tokenStream
/// </param>
/// <param name="">text
/// </param>
/// <param name="">maxNumFragments
/// </param>
/// <param name="">mergeContiguousFragments
/// </param>
/// <throws> IOException </throws>
public TextFragment[] GetBestTextFragments(TokenStream tokenStream, string text, bool mergeContiguousFragments, int maxNumFragments)
{
ArrayList docFrags = new ArrayList();
StringBuilder newText = new StringBuilder();
TextFragment currentFrag = new TextFragment(newText, newText.Length, docFrags.Count);
fragmentScorer.StartFragment(currentFrag);
docFrags.Add(currentFrag);
FragmentQueue fragQueue = new FragmentQueue(maxNumFragments);
try
{
Lucene.Net.Analysis.Token token;
string tokenText;
int startOffset;
int endOffset;
int lastEndOffset = 0;
textFragmenter.Start(text);
TokenGroup tokenGroup = new TokenGroup();
while ((token = tokenStream.Next()) != null)
{
if ((tokenGroup.numTokens > 0) && (tokenGroup.IsDistinct(token)))
{
//the current token is distinct from previous tokens -
// markup the cached token group info
startOffset = tokenGroup.startOffset;
endOffset = tokenGroup.endOffset;
tokenText = text.Substring(startOffset, (endOffset) - (startOffset));
string markedUpText = formatter.HighlightTerm(encoder.EncodeText(tokenText), tokenGroup);
//store any whitespace etc from between this and last group
if (startOffset > lastEndOffset)
newText.Append(encoder.EncodeText(text.Substring(lastEndOffset, (startOffset) - (lastEndOffset))));
newText.Append(markedUpText);
lastEndOffset = endOffset;
tokenGroup.Clear();
//check if current token marks the start of a new fragment
if (textFragmenter.IsNewFragment(token))
{
currentFrag.SetScore(fragmentScorer.FragmentScore);
//record stats for a new fragment
currentFrag.textEndPos = newText.Length;
currentFrag = new TextFragment(newText, newText.Length, docFrags.Count);
fragmentScorer.StartFragment(currentFrag);
docFrags.Add(currentFrag);
}
}
tokenGroup.AddToken(token, fragmentScorer.GetTokenScore(token));
if (lastEndOffset > maxDocBytesToAnalyze)
{
break;
}
}
currentFrag.SetScore(fragmentScorer.FragmentScore);
if (tokenGroup.numTokens > 0)
{
//flush the accumulated text (same code as in above loop)
startOffset = tokenGroup.startOffset;
endOffset = tokenGroup.endOffset;
tokenText = text.Substring(startOffset, (endOffset) - (startOffset));
string markedUpText = formatter.HighlightTerm(encoder.EncodeText(tokenText), tokenGroup);
//store any whitespace etc from between this and last group
if (startOffset > lastEndOffset)
newText.Append(encoder.EncodeText(text.Substring(lastEndOffset, (startOffset) - (lastEndOffset))));
newText.Append(markedUpText);
lastEndOffset = endOffset;
}
// append text after end of last token
// if (lastEndOffset < text.length())
// newText.append(encoder.encodeText(text.substring(lastEndOffset)));
currentFrag.textEndPos = newText.Length;
//sort the most relevant sections of the text
for (IEnumerator i = docFrags.GetEnumerator(); i.MoveNext(); )
{
currentFrag = (TextFragment) i.Current;
//If you are running with a version of Lucene before 11th Sept 03
// you do not have PriorityQueue.insert() - so uncomment the code below
/*
if (currentFrag.getScore() >= minScore)
{
fragQueue.put(currentFrag);
if (fragQueue.size() > maxNumFragments)
{ // if hit queue overfull
fragQueue.pop(); // remove lowest in hit queue
minScore = ((TextFragment) fragQueue.top()).getScore(); // reset minScore
}
}
*/
//The above code caused a problem as a result of Christoph Goller's 11th Sept 03
//fix to PriorityQueue. The correct method to use here is the new "insert" method
// USE ABOVE CODE IF THIS DOES NOT COMPILE!
fragQueue.Insert(currentFrag);
}
//return the most relevant fragments
TextFragment[] frag = new TextFragment[fragQueue.Size()];
for (int i = frag.Length - 1; i >= 0; i--)
{
frag[i] = (TextFragment) fragQueue.Pop();
}
//merge any contiguous fragments to improve readability
if (mergeContiguousFragments)
{
MergeContiguousFragments(frag);
ArrayList fragTexts = new ArrayList();
for (int i = 0; i < frag.Length; i++)
{
if ((frag[i] != null) && (frag[i].GetScore() > 0))
{
fragTexts.Add(frag[i]);
}
}
//frag = (TextFragment[]) ICollectionSupport.ToArray(fragTexts, new TextFragment[0]);
frag = (TextFragment[]) fragTexts.ToArray(typeof(TextFragment));
}
return frag;
}
finally
{
if (tokenStream != null)
{
try
{
tokenStream.Close();
}
catch (Exception e)
{
throw e;
}
}
}
}
public virtual void StartFragment(TextFragment newFragment)
{
uniqueTermsInFragment = new HashSet <string>();
currentTextFragment = newFragment;
totalScore = 0;
}
/// <summary></summary>
/// <param name="frag2">Fragment to be merged into this one</param>
public void Merge(TextFragment frag2)
{
TextEndPos = frag2.TextEndPos;
Score = Math.Max(Score, frag2.Score);
}
/// <summary>
/// true if this fragment follows the one passed
/// </summary>
public bool Follows(TextFragment fragment)
{
return TextStartPos == fragment.TextEndPos;
}
/*
* (non-Javadoc)
*
* @see
* org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
* .lucene.search.highlight.TextFragment)
*/
public void StartFragment(TextFragment newFragment)
{
uniqueTermsInFragment = new HashSet<String>();
currentTextFragment = newFragment;
totalScore = 0;
}
public virtual void StartFragment(TextFragment newFragment)
{
//uniqueTermsInFragment = new HashSetSupport();
uniqueTermsInFragment = new Hashtable();
currentTextFragment = newFragment;
totalScore = 0;
}
/// <summary>
/// Starts the fragment.
/// </summary>
/// <param name="newFragment">The new fragment.</param>
public void StartFragment(TextFragment newFragment)
{
foundTerms = new HashSet<string>();
totalScore = 0;
}