public override IList <LookupResult> Lookup(string key, HashSet <BytesRef> contexts, bool onlyMorePopular, int num)
{
Debug.Assert(num > 0);
if (onlyMorePopular)
{
throw new System.ArgumentException("this suggester only works with onlyMorePopular=false");
}
if (contexts != null)
{
throw new System.ArgumentException("this suggester doesn't support contexts");
}
if (fst == null)
{
return(Collections.emptyList());
}
//System.out.println("lookup key=" + key + " num=" + num);
for (var i = 0; i < key.Length; i++)
{
if (key[i] == 0x1E)
{
throw new ArgumentException(
"lookup key cannot contain HOLE character U+001E; this character is reserved");
}
if (key[i] == 0x1F)
{
throw new ArgumentException(
"lookup key cannot contain unit separator character U+001F; this character is reserved");
}
}
var utf8Key = new BytesRef(key);
try
{
Automaton lookupAutomaton = ToLookupAutomaton(key);
var spare = new CharsRef();
//System.out.println(" now intersect exactFirst=" + exactFirst);
// Intersect automaton w/ suggest wFST and get all
// prefix starting nodes & their outputs:
//final PathIntersector intersector = getPathIntersector(lookupAutomaton, fst);
//System.out.println(" prefixPaths: " + prefixPaths.size());
FST.BytesReader bytesReader = fst.BytesReader;
FST.Arc <PairOutputs.Pair <long?, BytesRef> > scratchArc = new FST.Arc <PairOutputs.Pair <long?, BytesRef> >();
IList <LookupResult> results = new List <LookupResult>();
IList <FSTUtil.Path <PairOutputs.Pair <long?, BytesRef> > > prefixPaths =
FSTUtil.intersectPrefixPaths(convertAutomaton(lookupAutomaton), fst);
if (exactFirst)
{
int count = 0;
foreach (FSTUtil.Path <PairOutputs.Pair <long?, BytesRef> > path in prefixPaths)
{
if (fst.FindTargetArc(END_BYTE, path.fstNode, scratchArc, bytesReader) != null)
{
// This node has END_BYTE arc leaving, meaning it's an
// "exact" match:
count++;
}
}
// Searcher just to find the single exact only
// match, if present:
Util.TopNSearcher <PairOutputs.Pair <long?, BytesRef> > searcher;
searcher = new Util.TopNSearcher <>(fst, count * maxSurfaceFormsPerAnalyzedForm,
count * maxSurfaceFormsPerAnalyzedForm, weightComparator);
// NOTE: we could almost get away with only using
// the first start node. The only catch is if
// maxSurfaceFormsPerAnalyzedForm had kicked in and
// pruned our exact match from one of these nodes
// ...:
foreach (FSTUtil.Path <PairOutputs.Pair <long?, BytesRef> > path in prefixPaths)
{
if (fst.FindTargetArc(END_BYTE, path.fstNode, scratchArc, bytesReader) != null)
{
// This node has END_BYTE arc leaving, meaning it's an
// "exact" match:
searcher.AddStartPaths(scratchArc, fst.outputs.add(path.output, scratchArc.Output), false,
path.input);
}
}
Util.TopResults <PairOutputs.Pair <long?, BytesRef> > completions = searcher.Search();
Debug.Assert(completions.IsComplete);
// NOTE: this is rather inefficient: we enumerate
// every matching "exactly the same analyzed form"
// path, and then do linear scan to see if one of
// these exactly matches the input. It should be
// possible (though hairy) to do something similar
// to getByOutput, since the surface form is encoded
// into the FST output, so we more efficiently hone
// in on the exact surface-form match. Still, I
// suspect very little time is spent in this linear
// seach: it's bounded by how many prefix start
// nodes we have and the
// maxSurfaceFormsPerAnalyzedForm:
foreach (Util.Result <PairOutputs.Pair <long?, BytesRef> > completion in completions)
{
BytesRef output2 = completion.Output.output2;
if (SameSurfaceForm(utf8Key, output2))
{
results.Add(GetLookupResult(completion.Output.output1, output2, spare));
break;
}
}
if (results.Count == num)
{
// That was quick:
return(results);
}
}
Util.TopNSearcher <PairOutputs.Pair <long?, BytesRef> > searcher;
searcher = new TopNSearcherAnonymousInnerClassHelper(this, fst, num - results.Count,
num * maxAnalyzedPathsForOneInput, weightComparator, utf8Key, results);
prefixPaths = getFullPrefixPaths(prefixPaths, lookupAutomaton, fst);
foreach (FSTUtil.Path <PairOutputs.Pair <long?, BytesRef> > path in prefixPaths)
{
searcher.AddStartPaths(path.fstNode, path.output, true, path.input);
}
Util.TopResults <PairOutputs.Pair <long?, BytesRef> > completions = searcher.search();
Debug.Assert(completions.IsComplete);
foreach (Util.Result <PairOutputs.Pair <long?, BytesRef> > completion in completions)
{
LookupResult result = GetLookupResult(completion.Output.output1, completion.Output.output2, spare);
// TODO: for fuzzy case would be nice to return
// how many edits were required
//System.out.println(" result=" + result);
results.Add(result);
if (results.Count == num)
{
// In the exactFirst=true case the search may
// produce one extra path
break;
}
}
return(results);
}
catch (IOException bogus)
{
throw new Exception(bogus);
}
}
/// <summary>
/// Retrieve suggestions.
/// </summary>
public virtual IList <LookupResult> Lookup(string key, HashSet <BytesRef> contexts, int num)
{
if (contexts != null)
{
throw new System.ArgumentException("this suggester doesn't support contexts");
}
TokenStream ts = queryAnalyzer.TokenStream("", key.ToString());
try
{
TermToBytesRefAttribute termBytesAtt = ts.AddAttribute <TermToBytesRefAttribute>();
OffsetAttribute offsetAtt = ts.AddAttribute <OffsetAttribute>();
PositionLengthAttribute posLenAtt = ts.AddAttribute <PositionLengthAttribute>();
PositionIncrementAttribute posIncAtt = ts.AddAttribute <PositionIncrementAttribute>();
ts.Reset();
var lastTokens = new BytesRef[grams];
//System.out.println("lookup: key='" + key + "'");
// Run full analysis, but save only the
// last 1gram, last 2gram, etc.:
BytesRef tokenBytes = termBytesAtt.BytesRef;
int maxEndOffset = -1;
bool sawRealToken = false;
while (ts.IncrementToken())
{
termBytesAtt.FillBytesRef();
sawRealToken |= tokenBytes.Length > 0;
// TODO: this is somewhat iffy; today, ShingleFilter
// sets posLen to the gram count; maybe we should make
// a separate dedicated att for this?
int gramCount = posLenAtt.PositionLength;
Debug.Assert(gramCount <= grams);
// Safety: make sure the recalculated count "agrees":
if (CountGrams(tokenBytes) != gramCount)
{
throw new System.ArgumentException("tokens must not contain separator byte; got token=" + tokenBytes + " but gramCount=" + gramCount + " does not match recalculated count=" + countGrams(tokenBytes));
}
maxEndOffset = Math.Max(maxEndOffset, offsetAtt.EndOffset());
lastTokens[gramCount - 1] = BytesRef.DeepCopyOf(tokenBytes);
}
ts.End();
if (!sawRealToken)
{
throw new System.ArgumentException("no tokens produced by analyzer, or the only tokens were empty strings");
}
// Carefully fill last tokens with _ tokens;
// ShingleFilter appraently won't emit "only hole"
// tokens:
int endPosInc = posIncAtt.PositionIncrement;
// Note this will also be true if input is the empty
// string (in which case we saw no tokens and
// maxEndOffset is still -1), which in fact works out OK
// because we fill the unigram with an empty BytesRef
// below:
bool lastTokenEnded = offsetAtt.EndOffset() > maxEndOffset || endPosInc > 0;
//System.out.println("maxEndOffset=" + maxEndOffset + " vs " + offsetAtt.endOffset());
if (lastTokenEnded)
{
//System.out.println(" lastTokenEnded");
// If user hit space after the last token, then
// "upgrade" all tokens. This way "foo " will suggest
// all bigrams starting w/ foo, and not any unigrams
// starting with "foo":
for (int i = grams - 1; i > 0; i--)
{
BytesRef token = lastTokens[i - 1];
if (token == null)
{
continue;
}
token.Grow(token.Length + 1);
token.Bytes[token.Length] = separator;
token.Length++;
lastTokens[i] = token;
}
lastTokens[0] = new BytesRef();
}
FST.Arc <long?> arc = new FST.Arc <long?>();
FST.BytesReader bytesReader = fst.BytesReader;
// Try highest order models first, and if they return
// results, return that; else, fallback:
double backoff = 1.0;
IList <LookupResult> results = new List <LookupResult>(num);
// We only add a given suffix once, from the highest
// order model that saw it; for subsequent lower order
// models we skip it:
var seen = new HashSet <BytesRef>();
for (int gram = grams - 1; gram >= 0; gram--)
{
BytesRef token = lastTokens[gram];
// Don't make unigram predictions from empty string:
if (token == null || (token.Length == 0 && key.Length > 0))
{
// Input didn't have enough tokens:
//System.out.println(" gram=" + gram + ": skip: not enough input");
continue;
}
if (endPosInc > 0 && gram <= endPosInc)
{
// Skip hole-only predictions; in theory we
// shouldn't have to do this, but we'd need to fix
// ShingleFilter to produce only-hole tokens:
//System.out.println(" break: only holes now");
break;
}
//System.out.println("try " + (gram+1) + " gram token=" + token.utf8ToString());
// TODO: we could add fuzziness here
// match the prefix portion exactly
//Pair<Long,BytesRef> prefixOutput = null;
long?prefixOutput = null;
try
{
prefixOutput = LookupPrefix(fst, bytesReader, token, arc);
}
catch (IOException bogus)
{
throw new Exception(bogus);
}
//System.out.println(" prefixOutput=" + prefixOutput);
if (prefixOutput == null)
{
// This model never saw this prefix, e.g. the
// trigram model never saw context "purple mushroom"
backoff *= ALPHA;
continue;
}
// TODO: we could do this division at build time, and
// bake it into the FST?
// Denominator for computing scores from current
// model's predictions:
long contextCount = totTokens;
BytesRef lastTokenFragment = null;
for (int i = token.Length - 1; i >= 0; i--)
{
if (token.Bytes[token.Offset + i] == separator)
{
BytesRef context = new BytesRef(token.Bytes, token.Offset, i);
long? output = Util.Get(fst, Util.ToIntsRef(context, new IntsRef()));
Debug.Assert(output != null);
contextCount = DecodeWeight(output);
lastTokenFragment = new BytesRef(token.Bytes, token.Offset + i + 1, token.Length - i - 1);
break;
}
}
BytesRef finalLastToken;
if (lastTokenFragment == null)
{
finalLastToken = BytesRef.DeepCopyOf(token);
}
else
{
finalLastToken = BytesRef.DeepCopyOf(lastTokenFragment);
}
Debug.Assert(finalLastToken.Offset == 0);
CharsRef spare = new CharsRef();
// complete top-N
Util.TopResults <long?> completions = null;
try
{
// Because we store multiple models in one FST
// (1gram, 2gram, 3gram), we must restrict the
// search so that it only considers the current
// model. For highest order model, this is not
// necessary since all completions in the FST
// must be from this model, but for lower order
// models we have to filter out the higher order
// ones:
// Must do num+seen.size() for queue depth because we may
// reject up to seen.size() paths in acceptResult():
Util.TopNSearcher <long?> searcher = new TopNSearcherAnonymousInnerClassHelper(this, fst, num, num + seen.Count, weightComparator, seen, finalLastToken);
// since this search is initialized with a single start node
// it is okay to start with an empty input path here
searcher.AddStartPaths(arc, prefixOutput, true, new IntsRef());
completions = searcher.Search();
Debug.Assert(completions.IsComplete);
}
catch (IOException bogus)
{
throw new Exception(bogus);
}
int prefixLength = token.Length;
BytesRef suffix = new BytesRef(8);
//System.out.println(" " + completions.length + " completions");
foreach (Util.Result <long?> completion in completions)
{
token.Length = prefixLength;
// append suffix
Util.ToBytesRef(completion.Input, suffix);
token.Append(suffix);
//System.out.println(" completion " + token.utf8ToString());
// Skip this path if a higher-order model already
// saw/predicted its last token:
BytesRef lastToken = token;
for (int i = token.Length - 1; i >= 0; i--)
{
if (token.Bytes[token.Offset + i] == separator)
{
Debug.Assert(token.Length - i - 1 > 0);
lastToken = new BytesRef(token.Bytes, token.Offset + i + 1, token.Length - i - 1);
break;
}
}
if (seen.Contains(lastToken))
{
//System.out.println(" skip dup " + lastToken.utf8ToString());
goto nextCompletionContinue;
}
seen.Add(BytesRef.DeepCopyOf(lastToken));
spare.Grow(token.Length);
UnicodeUtil.UTF8toUTF16(token, spare);
LookupResult result = new LookupResult(spare.ToString(), (long)(long.MaxValue * backoff * ((double)decodeWeight(completion.Output)) / contextCount));
results.Add(result);
Debug.Assert(results.Count == seen.Count);
//System.out.println(" add result=" + result);
nextCompletionContinue :;
}
nextCompletionBreak :
backoff *= ALPHA;
}
results.Sort(new ComparatorAnonymousInnerClassHelper(this));
if (results.Count > num)
{
results.SubList(num, results.Count).Clear();
}
return(results);
}
finally
{
IOUtils.CloseWhileHandlingException(ts);
}
}