private int Advance(string source, Span span)
{
var offset = span.Start + span.Length;
while (offset < span.Length && char.IsWhiteSpace(source, offset))
{
offset++;
}
return offset;
}
public Token(TokenType tokenType, string source, Span span)
{
this.tokenType = tokenType;
if (source == null)
{
throw new ArgumentNullException("source");
}
this.value = source.Substring(span.Start, span.Length);
if (span == null)
{
throw new ArgumentNullException("span");
}
this.span = span;
}
private Token Accept(TokenType tokenType, Cursor cursor)
{
var terminal = this.terminals[tokenType];
var length = terminal(cursor.Source, cursor.Offset);
if (length == null)
{
return null;
}
var span = new Span(cursor.Offset, length.Value);
cursor.Offset = this.Advance(cursor.Source, span);
return new Token(tokenType, cursor.Source, span);
}