private void ProcessMultiLine(CandidateDefinition matchingToken)
{
var multiLineType = _current.Type;
var multiLineToken = new System.Text.StringBuilder(_current.Value);
Match continuation;
do
{
continuation = matchingToken.Definition.MultiLineContinuation.Match(_line);
if (continuation.Value.Length > 0)
{
AdvanceCurrentToken(
continuation,
matchingToken.Definition.Type
);
var linesConsumed = ReadNextLine();
multiLineToken.Append(_current.Value);
for (int i = 0; i < linesConsumed; i++)
multiLineToken.AppendLine();
}
}
while (HasMoreTokens && continuation != null && continuation.Value.Length > 0);
Match terminationMatch = matchingToken.Definition.MultiLineTerminator.Match(_line);
if (terminationMatch.Value.Length < 0)
throw new SyntaxException(String.Format("Failed to find terminal for {0}", matchingToken.Definition.Type));
AdvanceCurrentToken(terminationMatch, matchingToken.Definition.Type);
multiLineToken.Append(_current.Value);
_current = new Token(multiLineToken.ToString(), multiLineType);
}
private int ReadNextLine()
{
int linesConsumed = 0;
while (_line.Length == 0 && _reader.Peek() != -1)
{
if (Position.Column > 1)
_current = new Token { Type = TokenType.NewLine };
_line = _reader.ReadLine();
Position.NewRow();
linesConsumed++;
}
return linesConsumed;
}
private void AdvanceCurrentToken(Match match, TokenType type)
{
_current = new Token(match.Value, type);
_line = _line.Remove(match.Captures[0].Index, match.Captures[0].Length);
Position.Column += match.Value.Length;
}
public override void ReadNextToken()
{
_current = null;
ReadNextLine();
if (_current != Token.Null && _current.Type == TokenType.NewLine)
return;
if (!HasMoreTokens)
return;
var m = TokenDefinitions
.Select((td, i) => new CandidateDefinition(i, td.Regex.Match(_line), td))
.ToList();
var matchingToken = GetCandidateDefinition(m);
if (matchingToken == null)
throw new SyntaxException();
if (matchingToken.Definition.IsMultiLine)
ProcessMultiLine(matchingToken);
if (Current == Token.Null)
return;
bool skippable;
if ((_skippedTokens.TryGetValue(Current.Type, out skippable)) && skippable)
ReadNextToken();
}