private IToken AnalyzeElement(Context context, Action<ITokenRecognizer> action)
{
IToken ret = null;
var possible = _recognizers.Where(x => !x.IsInEndState());
possible.ForEach(x => action(x));
possible = _recognizers.Where(x => x.State == RecognizerState.Possible);
var recognized = _recognizers.Where(x => x.State == RecognizerState.Recognized
&& (x.PreviousState == RecognizerState.Possible || x.PreviousState == RecognizerState.Start));
if (recognized.Count() > 1)
{
ThrowAmbiguousTokenException(context.Location, recognized);
}
else if (recognized.Count() == 1)
{
context.LongestRecognizedToken = recognized.First().GetToken(context);
}
if (possible.Count() == 0)
{
if (context.LongestRecognizedToken != null)
ret = NewToken(context);
else
throw new UnexpectedCharacterException($"Unexpected character: {context.Section.Text[context.StartIdx]}");
}
if (ret == null)
context.CurrentIdx++;
return ret;
}
public IEnumerable<IToken> GetTokens(IEnumerable<ISection> sections)
{
var context = new Context();
foreach (var s in sections)
{
IToken token = null;
context.Section = s;
context.Location = context.Section.CreateLocation(context.CurrentIdx);
while (context.CurrentIdx < s.Text.Length)
{
token = AnalyzeChar(context, s.Text[context.CurrentIdx]);
if (token != null)
yield return token;
}
token = AnalyzeEndOfSection(context);
if (token != null)
yield return token;
if (token == null)
{
throw new UnexpectedCharacterException($"Unexpected character: {s.Text[context.StartIdx]}");
}
}
}
private IToken NewToken(Context context)
{
context.StartIdx += context.LongestRecognizedToken.Length;
context.CurrentIdx = context.StartIdx;
context.Location = context.Section.CreateLocation(context.CurrentIdx);
_recognizers.ForEach(x => x.ResetState());
return context.LongestRecognizedToken;
}
private IToken AnalyzeChar(Context context, char c)
{
return AnalyzeElement(context, x => x.Feed(c));
}
private IToken AnalyzeEndOfSection(Context context)
{
return AnalyzeElement(context, x => x.FeedEndOfSection());
}