public override void ReadNextToken()
{
_current = null;
ReadNextLine();
if (!HasMoreTokens)
return;
var m = TokenDefinitions
.Select((td, i) => new CandidateDefinition(i, td.Regex.Match(_line), td))
.ToList();
var matchingToken = GetCandidateDefinition(m);
if (matchingToken == null)
throw new SyntaxException();
if (matchingToken.Definition.IsMultiLine)
ProcessMultiLine(matchingToken);
if (matchingToken.Definition.Skip)
ReadNextToken();
}
public override void ReadNextToken()
{
_current = null;
var current = new StringBuilder();
TokenDefinition definition;
TokenDefinition last;
int next;
while ((next = _reader.Peek()) != -1)
{
int matchCount = MatchCount(current.ToString() + (char)next, out definition);
last = definition;
int lastMatchCount = 0;
while (_reader.Peek() != -1 && (matchCount > 0 || (lastMatchCount == 0)))
{
last = definition;
ReadChar();
current.Append((char)next);
next = _reader.Peek();
if (next == -1)
break;
lastMatchCount = matchCount;
matchCount = MatchCount(current.ToString() + (char)next, out definition);
}
if (last != null && last.Skip && !IsQuote(_currentQuoteType))
{
ReadNextToken();
return;
}
// if we get here, we either have a good token, or the token is not recognised
if (MatchCount(current.ToString(), out definition) == 0)
throw new UnknownTokenException(current.ToString());
_current = new Token(current.ToString(), last != null ? last.Type : TokenType.None);
if (IsQuote(_current.Type))
{
if (_currentQuoteType != _current.Type)
_currentQuoteType = _current.Type;
else
_currentQuoteType = TokenType.None;
}
return;
}
}
private void ProcessMultiLine(CandidateDefinition matchingToken)
{
var multiLineType = _current.Type;
var multiLineToken = new System.Text.StringBuilder(_current.Value);
Match continuation;
do
{
continuation = matchingToken.Definition.MultiLineContinuation.Match(_line);
if (continuation.Value.Length > 0)
{
AdvanceCurrentToken(
continuation,
matchingToken.Definition.Type
);
var linesConsumed = ReadNextLine();
multiLineToken.Append(_current.Value);
for (int i = 0; i < linesConsumed; i++)
multiLineToken.AppendLine();
}
}
while (HasMoreTokens && continuation != null && continuation.Value.Length > 0);
Match terminationMatch = matchingToken.Definition.MultiLineTerminator.Match(_line);
if (terminationMatch.Value.Length < 0)
throw new SyntaxException(String.Format("Failed to find terminal for {0}", matchingToken.Definition.Type));
AdvanceCurrentToken(terminationMatch, matchingToken.Definition.Type);
multiLineToken.Append(_current.Value);
_current = new Token(multiLineToken.ToString(), multiLineType);
}
private void AdvanceCurrentToken(Match match, TokenType type)
{
_current = new Token(match.Value, type);
_line = _line.Remove(match.Captures[0].Index, match.Captures[0].Length);
Position.Column += match.Value.Length;
}