private void EvaluateAndSortCandidates(List <Core.Tokenization.PrioritizedToken> candidates)
{
if (candidates == null || candidates.Count == 0)
{
return;
}
int basePriority = _Priority;
foreach (Core.Tokenization.PrioritizedToken pt in candidates)
{
Core.Tokenization.NumberToken nt = pt.Token as Core.Tokenization.NumberToken;
System.Diagnostics.Debug.Assert(nt != null);
int malus = 0;
if (nt.DecimalSeparator == NumericSeparator.Alternate ||
nt.GroupSeparator == NumericSeparator.Alternate)
{
++malus;
}
pt.Priority = _Priority - malus;
}
candidates.Sort(delegate(PrioritizedToken a, PrioritizedToken b)
{
// sort by decreasing priority
return(b.Priority - a.Priority);
});
}
public override Core.Tokenization.Token Recognize(string s,
int from, bool allowTokenBundles, ref int consumedLength)
{
consumedLength = 0;
List <FSTMatch> matches = _FSTRecognizer.ComputeMatches(s, from, false, 0, true);
if (matches == null || matches.Count == 0)
{
return(null);
}
List <Core.Tokenization.PrioritizedToken> candidates = null;
foreach (FSTMatch m in matches)
{
System.Diagnostics.Debug.Assert(m.Index == from);
if (VerifyContextConstraints(s, m.Index + m.Length))
{
if (m.Output == null || m.Output.Length != m.Length || m.Length == 0)
{
throw new Exception("Internal error: invalid number FST");
}
if (consumedLength == 0)
{
consumedLength = m.Length;
}
else
{
System.Diagnostics.Debug.Assert(consumedLength == m.Length);
}
Core.Tokenization.NumberToken nt = ParseNumber(s.Substring(m.Index, m.Length),
m.Output);
if (nt != null)
{
if (candidates == null)
{
candidates = new List <PrioritizedToken>();
}
candidates.Add(new Core.Tokenization.PrioritizedToken(nt, 0));
}
else
{
System.Diagnostics.Debug.Assert(false, "Expect to be able to parse a number if returned by the FST");
}
}
}
if (candidates == null || candidates.Count == 0)
{
return(null);
}
EvaluateAndSortCandidates(candidates);
if (allowTokenBundles && candidates.Count > 1)
{
return(new Core.Tokenization.TokenBundle(candidates));
}
else
{
return(candidates[0].Token);
}
}