public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
var leftPart = LexInteger(stream);
var postfix = GetPostfix(stream);
if (postfix != null)
{
return new Token(TokenType.Float, $"{leftPart}{postfix}");
}
if (stream.Current != ".")
{
stream.RollbackSnapshot();
return null;
}
stream.Consume();
var rightPart = LexInteger(stream);
if (string.IsNullOrEmpty(rightPart))
{
stream.RollbackSnapshot();
return null;
}
postfix = GetPostfix(stream) ?? string.Empty;
stream.CommitSnapshot();
return new Token(TokenType.Float, $"{leftPart}.{rightPart}{postfix}");
}
public override Token GetToken(TransactableTokenStream stream)
{
if (stream.Current != "/")
{
return null;
}
stream.TakeSnapshot();
stream.Consume();
string comment;
switch (stream.Current)
{
case "/":
stream.Consume();
comment = LexSingleLineComment(stream);
break;
case "*":
stream.Consume();
comment = LexMultiLineComment(stream);
break;
default:
stream.RollbackSnapshot();
return null;
}
stream.CommitSnapshot();
return new Token(TokenType.Comment, comment);
}
public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
if (stream.Current == null || char.IsNumber(stream.Current[0]) || !char.IsLetter(stream.Current[0]))
{
stream.RollbackSnapshot();
return null;
}
var symbolName = string.Empty;
while (stream.Current != null && (char.IsLetter(stream.Current[0]) || char.IsNumber(stream.Current[0])))
{
symbolName += stream.Current[0];
stream.Consume();
}
if (_forbiddenSymbolNames.Contains(symbolName))
{
stream.RollbackSnapshot();
return null;
}
stream.CommitSnapshot();
return new Token(TokenType.Identifier, symbolName);
}
public override Token GetToken(TransactableTokenStream stream)
{
if (stream.Current != "\"")
{
return null;
}
stream.TakeSnapshot();
var tokenValue = stream.Current;
stream.Consume();
while (stream.Current != null && stream.Current != "\"")
{
tokenValue += stream.Current;
stream.Consume();
if (stream.Current == null)
{
stream.RollbackSnapshot();
return null;
}
}
tokenValue += stream.Current;
stream.Consume();
stream.CommitSnapshot();
return new Token(TokenType.String, tokenValue);
}
public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
foreach (var character in _syntaxLiteral)
{
if (stream.Current == null || stream.Current != character.ToString())
{
stream.RollbackSnapshot();
return null;
}
stream.Consume();
}
stream.CommitSnapshot();
return new Token(_keyword, _syntaxLiteral);
}