/// <summary>
/// Constructor for enumeration of all terms from specified <c>reader</c> which share a prefix of
/// length <paramref name="prefixLength"/> with <paramref name="term"/> and which have a fuzzy similarity >
/// <paramref name="minSimilarity"/>.
/// <para/>
/// After calling the constructor the enumeration is already pointing to the first
/// valid term if such a term exists.
/// </summary>
/// <param name="terms"> Delivers terms. </param>
/// <param name="atts"> <see cref="AttributeSource"/> created by the rewrite method of <see cref="MultiTermQuery"/>
/// thats contains information about competitive boosts during rewrite. It is also used
/// to cache DFAs between segment transitions. </param>
/// <param name="term"> Pattern term. </param>
/// <param name="minSimilarity"> Minimum required similarity for terms from the reader. Pass an integer value
/// representing edit distance. Passing a fraction is deprecated. </param>
/// <param name="prefixLength"> Length of required common prefix. Default value is 0. </param>
/// <param name="transpositions"> Transpositions </param>
/// <exception cref="IOException"> if there is a low-level IO error </exception>
public FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, float minSimilarity, int prefixLength, bool transpositions)
{
InitializeInstanceFields();
if (minSimilarity >= 1.0f && minSimilarity != (int)minSimilarity)
{
throw new ArgumentException("fractional edit distances are not allowed");
}
if (minSimilarity < 0.0f)
{
throw new ArgumentException("minimumSimilarity cannot be less than 0");
}
if (prefixLength < 0)
{
throw new ArgumentException("prefixLength cannot be less than 0");
}
this.m_terms = terms;
this.term = term;
// convert the string into a utf32 int[] representation for fast comparisons
string utf16 = term.Text();
this.m_termText = new int[utf16.CodePointCount(0, utf16.Length)];
for (int cp, i = 0, j = 0; i < utf16.Length; i += Character.CharCount(cp))
{
m_termText[j++] = cp = utf16.CodePointAt(i);
}
this.m_termLength = m_termText.Length;
this.dfaAtt = atts.AddAttribute <ILevenshteinAutomataAttribute>();
//The prefix could be longer than the word.
//It's kind of silly though. It means we must match the entire word.
this.m_realPrefixLength = prefixLength > m_termLength ? m_termLength : prefixLength;
// if minSimilarity >= 1, we treat it as number of edits
if (minSimilarity >= 1f)
{
this.m_minSimilarity = 0; // just driven by number of edits
m_maxEdits = (int)minSimilarity;
m_raw = true;
}
else
{
this.m_minSimilarity = minSimilarity;
// calculate the maximum k edits for this similarity
m_maxEdits = InitialMaxDistance(this.m_minSimilarity, m_termLength);
m_raw = false;
}
if (transpositions && m_maxEdits > LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE)
{
throw new NotSupportedException("with transpositions enabled, distances > " + LevenshteinAutomata.MAXIMUM_SUPPORTED_DISTANCE + " are not supported ");
}
this.transpositions = transpositions;
this.m_scaleFactor = 1.0f / (1.0f - this.m_minSimilarity);
this.maxBoostAtt = atts.AddAttribute <IMaxNonCompetitiveBoostAttribute>();
bottom = maxBoostAtt.MaxNonCompetitiveBoost;
bottomTerm = maxBoostAtt.CompetitiveTerm;
BottomChanged(null, true);
}
public override bool Accept(AttributeSource source)
{
if (typeAtt == null)
{
typeAtt = source.AddAttribute <ITypeAttribute>();
}
return(typeToMatch.Equals(typeAtt.Type));
}
public override bool Accept(AttributeSource source)
{
if (typeAtt is null)
{
typeAtt = source.AddAttribute <ITypeAttribute>();
}
//check to see if this is a Category
return(typeToMatch.Equals(typeAtt.Type, StringComparison.Ordinal));
}
public override bool Accept(AttributeSource source)
{
if (typeAtt == null)
{
typeAtt = source.AddAttribute <ITypeAttribute>();
}
//check to see if this is a Category
return(typeToMatch.Equals(typeAtt.Type));
}
public override bool Accept(AttributeSource source)
{
if (m_termAtt is null)
{
m_termAtt = source.AddAttribute <ICharTermAttribute>();
}
//We don't care about the date, just that we can parse it as a date
if (m_formats is null)
{
return(DateTime.TryParse(m_termAtt.ToString(), m_culture, m_style, out _));
}
else
{
return(DateTime.TryParseExact(m_termAtt.ToString(), m_formats, m_culture, m_style, out _));
}
}
public override bool Accept(AttributeSource source)
{
if (termAtt == null)
{
termAtt = source.AddAttribute <ICharTermAttribute>();
}
DateTime date; //We don't care about the date, just that we can parse it as a date
if (formats == null)
{
return(DateTime.TryParse(termAtt.ToString(), culture, style, out date));
}
else
{
return(DateTime.TryParseExact(termAtt.ToString(), formats, culture, style, out date));
}
}
public override bool Accept(AttributeSource source)
{
if (termAtt == null)
{
termAtt = source.AddAttribute <ITermAttribute>();
}
try
{
DateTime date = DateTime.Parse(termAtt.Term, dateFormat);//We don't care about the date, just that we can parse it as a date
if (date != null)
{
return(true);
}
}
catch (FormatException)
{
}
return(false);
}
private void AddTerms(IndexReader reader, FieldVals f)
{
if (f.queryString is null)
{
return;
}
Terms terms = MultiFields.GetTerms(reader, f.fieldName);
if (terms is null)
{
return;
}
TokenStream ts = analyzer.GetTokenStream(f.fieldName, f.queryString);
try
{
ICharTermAttribute termAtt = ts.AddAttribute <ICharTermAttribute>();
int corpusNumDocs = reader.NumDocs;
ISet <string> processedTerms = new JCG.HashSet <string>();
ts.Reset();
while (ts.IncrementToken())
{
string term = termAtt.ToString();
if (!processedTerms.Contains(term))
{
processedTerms.Add(term);
ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
float minScore = 0;
Term startTerm = new Term(f.fieldName, term);
AttributeSource atts = new AttributeSource();
IMaxNonCompetitiveBoostAttribute maxBoostAtt =
atts.AddAttribute <IMaxNonCompetitiveBoostAttribute>();
#pragma warning disable 612, 618
SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
#pragma warning restore 612, 618
//store the df so all variants use same idf
int df = reader.DocFreq(startTerm);
int numVariants = 0;
int totalVariantDocFreqs = 0;
BytesRef possibleMatch;
IBoostAttribute boostAtt =
fe.Attributes.AddAttribute <IBoostAttribute>();
while (fe.MoveNext())
{
possibleMatch = fe.Term;
numVariants++;
totalVariantDocFreqs += fe.DocFreq;
float score = boostAtt.Boost;
if (variantsQ.Count < MAX_VARIANTS_PER_TERM || score > minScore)
{
ScoreTerm st = new ScoreTerm(new Term(startTerm.Field, BytesRef.DeepCopyOf(possibleMatch)), score, startTerm);
variantsQ.InsertWithOverflow(st);
minScore = variantsQ.Top.Score; // maintain minScore
}
maxBoostAtt.MaxNonCompetitiveBoost = variantsQ.Count >= MAX_VARIANTS_PER_TERM ? minScore : float.NegativeInfinity;
}
if (numVariants > 0)
{
int avgDf = totalVariantDocFreqs / numVariants;
if (df == 0) //no direct match we can use as df for all variants
{
df = avgDf; //use avg df of all variants
}
// take the top variants (scored by edit distance) and reset the score
// to include an IDF factor then add to the global queue for ranking
// overall top query terms
int size = variantsQ.Count;
for (int i = 0; i < size; i++)
{
ScoreTerm st = variantsQ.Pop();
st.Score = (st.Score * st.Score) * sim.Idf(df, corpusNumDocs);
q.InsertWithOverflow(st);
}
}
}
}
ts.End();
}
finally
{
IOUtils.DisposeWhileHandlingException(ts);
}
}
/// <summary>
/// Provide spelling corrections based on several parameters.
/// </summary>
/// <param name="term"> The term to suggest spelling corrections for </param>
/// <param name="numSug"> The maximum number of spelling corrections </param>
/// <param name="ir"> The index reader to fetch the candidate spelling corrections from </param>
/// <param name="docfreq"> The minimum document frequency a potential suggestion need to have in order to be included </param>
/// <param name="editDistance"> The maximum edit distance candidates are allowed to have </param>
/// <param name="accuracy"> The minimum accuracy a suggested spelling correction needs to have in order to be included </param>
/// <param name="spare"> a chars scratch </param>
/// <returns> a collection of spelling corrections sorted by <code>ScoreTerm</code>'s natural order. </returns>
/// <exception cref="System.IO.IOException"> If I/O related errors occur </exception>
protected internal virtual IEnumerable <ScoreTerm> SuggestSimilar(Term term, int numSug, IndexReader ir,
int docfreq, int editDistance, float accuracy, CharsRef spare)
{
var atts = new AttributeSource();
IMaxNonCompetitiveBoostAttribute maxBoostAtt = atts.AddAttribute <IMaxNonCompetitiveBoostAttribute>();
Terms terms = MultiFields.GetTerms(ir, term.Field);
if (terms == null)
{
return(new List <ScoreTerm>());
}
FuzzyTermsEnum e = new FuzzyTermsEnum(terms, atts, term, editDistance, Math.Max(minPrefix, editDistance - 1), true);
var stQueue = new Support.PriorityQueue <ScoreTerm>();
BytesRef queryTerm = new BytesRef(term.Text());
BytesRef candidateTerm;
ScoreTerm st = new ScoreTerm();
IBoostAttribute boostAtt = e.Attributes.AddAttribute <IBoostAttribute>();
while ((candidateTerm = e.Next()) != null)
{
float boost = boostAtt.Boost;
// ignore uncompetitive hits
if (stQueue.Count >= numSug && boost <= stQueue.Peek().Boost)
{
continue;
}
// ignore exact match of the same term
if (queryTerm.BytesEquals(candidateTerm))
{
continue;
}
int df = e.DocFreq;
// check docFreq if required
if (df <= docfreq)
{
continue;
}
float score;
string termAsString;
if (distance == INTERNAL_LEVENSHTEIN)
{
// delay creating strings until the end
termAsString = null;
// undo FuzzyTermsEnum's scale factor for a real scaled lev score
score = boost / e.ScaleFactor + e.MinSimilarity;
}
else
{
UnicodeUtil.UTF8toUTF16(candidateTerm, spare);
termAsString = spare.ToString();
score = distance.GetDistance(term.Text(), termAsString);
}
if (score < accuracy)
{
continue;
}
// add new entry in PQ
st.Term = BytesRef.DeepCopyOf(candidateTerm);
st.Boost = boost;
st.Docfreq = df;
st.TermAsString = termAsString;
st.Score = score;
stQueue.Offer(st);
// possibly drop entries from queue
st = (stQueue.Count > numSug) ? stQueue.Poll() : new ScoreTerm();
maxBoostAtt.MaxNonCompetitiveBoost = (stQueue.Count >= numSug) ? stQueue.Peek().Boost : float.NegativeInfinity;
}
return(stQueue);
}
/*
* Need to worry about multiple scenarios:
* - need to go for the longest match
* a b => foo #shouldn't match if "a b" is followed by "c d"
* a b c d => bar
* - need to backtrack - retry matches for tokens already read
* a b c d => foo
* b c => bar
* If the input stream is "a b c x", one will consume "a b c d"
* trying to match the first rule... all but "a" should be
* pushed back so a match may be made on "b c".
* - don't try and match generated tokens (thus need separate queue)
* matching is not recursive.
* - handle optional generation of original tokens in all these cases,
* merging token streams to preserve token positions.
* - preserve original positionIncrement of first matched token
*/
public override bool IncrementToken()
{
while (true)
{
// if there are any generated tokens, return them... don't try any
// matches against them, as we specifically don't want recursion.
if (replacement != null && replacement.MoveNext())
{
Copy(this, replacement.Current);
return(true);
}
// common case fast-path of first token not matching anything
AttributeSource firstTok = NextTok();
if (firstTok == null)
{
return(false);
}
var termAtt = firstTok.AddAttribute <ICharTermAttribute>();
SlowSynonymMap result = map.Submap != null?map.Submap.Get(termAtt.Buffer, 0, termAtt.Length) : null;
if (result == null)
{
Copy(this, firstTok);
return(true);
}
// fast-path failed, clone ourselves if needed
if (firstTok == this)
{
firstTok = CloneAttributes();
}
// OK, we matched a token, so find the longest match.
matched = new LinkedList <AttributeSource>();
result = Match(result);
if (result == null)
{
// no match, simply return the first token read.
Copy(this, firstTok);
return(true);
}
// reuse, or create new one each time?
IList <AttributeSource> generated = new JCG.List <AttributeSource>(result.Synonyms.Length + matched.Count + 1);
//
// there was a match... let's generate the new tokens, merging
// in the matched tokens (position increments need adjusting)
//
AttributeSource lastTok = matched.Count == 0 ? firstTok : matched.Last.Value;
bool includeOrig = result.IncludeOrig;
AttributeSource origTok = includeOrig ? firstTok : null;
IPositionIncrementAttribute firstPosIncAtt = firstTok.AddAttribute <IPositionIncrementAttribute>();
int origPos = firstPosIncAtt.PositionIncrement; // position of origTok in the original stream
int repPos = 0; // curr position in replacement token stream
int pos = 0; // current position in merged token stream
for (int i = 0; i < result.Synonyms.Length; i++)
{
Token repTok = result.Synonyms[i];
AttributeSource newTok = firstTok.CloneAttributes();
ICharTermAttribute newTermAtt = newTok.AddAttribute <ICharTermAttribute>();
IOffsetAttribute newOffsetAtt = newTok.AddAttribute <IOffsetAttribute>();
IPositionIncrementAttribute newPosIncAtt = newTok.AddAttribute <IPositionIncrementAttribute>();
IOffsetAttribute lastOffsetAtt = lastTok.AddAttribute <IOffsetAttribute>();
newOffsetAtt.SetOffset(newOffsetAtt.StartOffset, lastOffsetAtt.EndOffset);
newTermAtt.CopyBuffer(repTok.Buffer, 0, repTok.Length);
repPos += repTok.PositionIncrement;
if (i == 0) // make position of first token equal to original
{
repPos = origPos;
}
// if necessary, insert original tokens and adjust position increment
while (origTok != null && origPos <= repPos)
{
IPositionIncrementAttribute origPosInc = origTok.AddAttribute <IPositionIncrementAttribute>();
origPosInc.PositionIncrement = origPos - pos;
generated.Add(origTok);
pos += origPosInc.PositionIncrement;
//origTok = matched.Count == 0 ? null : matched.RemoveFirst();
if (matched.Count == 0)
{
origTok = null;
}
else
{
origTok = matched.First.Value;
matched.Remove(origTok);
}
if (origTok != null)
{
origPosInc = origTok.AddAttribute <IPositionIncrementAttribute>();
origPos += origPosInc.PositionIncrement;
}
}
newPosIncAtt.PositionIncrement = repPos - pos;
generated.Add(newTok);
pos += newPosIncAtt.PositionIncrement;
}
// finish up any leftover original tokens
while (origTok != null)
{
IPositionIncrementAttribute origPosInc = origTok.AddAttribute <IPositionIncrementAttribute>();
origPosInc.PositionIncrement = origPos - pos;
generated.Add(origTok);
pos += origPosInc.PositionIncrement;
if (matched.Count == 0)
{
origTok = null;
}
else
{
origTok = matched.First.Value;
matched.Remove(origTok);
}
if (origTok != null)
{
origPosInc = origTok.AddAttribute <IPositionIncrementAttribute>();
origPos += origPosInc.PositionIncrement;
}
}
// what if we replaced a longer sequence with a shorter one?
// a/0 b/5 => foo/0
// should I re-create the gap on the next buffered token?
replacement = generated.GetEnumerator();
// Now return to the top of the loop to read and return the first
// generated token.. The reason this is done is that we may have generated
// nothing at all, and may need to continue with more matching logic.
}
}
/// <summary>
/// <para>Get the next token from the input stream.
/// </para>
/// <para>If the next token has <c>positionIncrement > 1</c>,
/// <c>positionIncrement - 1</c> <see cref="fillerToken"/>s are
/// inserted first.
/// </para>
/// </summary>
/// <param name="target"> Where to put the new token; if null, a new instance is created. </param>
/// <returns> On success, the populated token; null otherwise </returns>
/// <exception cref="IOException"> if the input stream has a problem </exception>
private InputWindowToken GetNextToken(InputWindowToken target)
{
InputWindowToken newTarget = target;
if (numFillerTokensToInsert > 0)
{
if (null == target)
{
newTarget = new InputWindowToken(this, nextInputStreamToken.CloneAttributes());
}
else
{
nextInputStreamToken.CopyTo(target.attSource);
}
// A filler token occupies no space
newTarget.offsetAtt.SetOffset(newTarget.offsetAtt.StartOffset, newTarget.offsetAtt.StartOffset);
newTarget.termAtt.CopyBuffer(fillerToken, 0, fillerToken.Length);
newTarget.isFiller = true;
--numFillerTokensToInsert;
}
else if (isNextInputStreamToken)
{
if (null == target)
{
newTarget = new InputWindowToken(this, nextInputStreamToken.CloneAttributes());
}
else
{
nextInputStreamToken.CopyTo(target.attSource);
}
isNextInputStreamToken = false;
newTarget.isFiller = false;
}
else if (!exhausted)
{
if (m_input.IncrementToken())
{
if (null == target)
{
newTarget = new InputWindowToken(this, CloneAttributes());
}
else
{
this.CopyTo(target.attSource);
}
if (posIncrAtt.PositionIncrement > 1)
{
// Each output shingle must contain at least one input token,
// so no more than (maxShingleSize - 1) filler tokens will be inserted.
numFillerTokensToInsert = Math.Min(posIncrAtt.PositionIncrement - 1, maxShingleSize - 1);
// Save the current token as the next input stream token
if (null == nextInputStreamToken)
{
nextInputStreamToken = CloneAttributes();
}
else
{
this.CopyTo(nextInputStreamToken);
}
isNextInputStreamToken = true;
// A filler token occupies no space
newTarget.offsetAtt.SetOffset(offsetAtt.StartOffset, offsetAtt.StartOffset);
newTarget.termAtt.CopyBuffer(fillerToken, 0, fillerToken.Length);
newTarget.isFiller = true;
--numFillerTokensToInsert;
}
else
{
newTarget.isFiller = false;
}
}
else
{
exhausted = true;
m_input.End();
endState = CaptureState();
numFillerTokensToInsert = Math.Min(posIncrAtt.PositionIncrement, maxShingleSize - 1);
if (numFillerTokensToInsert > 0)
{
nextInputStreamToken = new AttributeSource(this.GetAttributeFactory());
nextInputStreamToken.AddAttribute <ICharTermAttribute>();
IOffsetAttribute newOffsetAtt = nextInputStreamToken.AddAttribute <IOffsetAttribute>();
newOffsetAtt.SetOffset(offsetAtt.EndOffset, offsetAtt.EndOffset);
// Recurse/loop just once:
return(GetNextToken(target));
}
else
{
newTarget = null;
}
}
}
else
{
newTarget = null;
}
return(newTarget);
}
