public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
if (stream.Current == null || char.IsNumber(stream.Current[0]) || !char.IsLetter(stream.Current[0]))
{
stream.RollbackSnapshot();
return null;
}
var symbolName = string.Empty;
while (stream.Current != null && (char.IsLetter(stream.Current[0]) || char.IsNumber(stream.Current[0])))
{
symbolName += stream.Current[0];
stream.Consume();
}
if (_forbiddenSymbolNames.Contains(symbolName))
{
stream.RollbackSnapshot();
return null;
}
stream.CommitSnapshot();
return new Token(TokenType.Identifier, symbolName);
}
public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
var leftPart = LexInteger(stream);
var postfix = GetPostfix(stream);
if (postfix != null)
{
return new Token(TokenType.Float, $"{leftPart}{postfix}");
}
if (stream.Current != ".")
{
stream.RollbackSnapshot();
return null;
}
stream.Consume();
var rightPart = LexInteger(stream);
if (string.IsNullOrEmpty(rightPart))
{
stream.RollbackSnapshot();
return null;
}
postfix = GetPostfix(stream) ?? string.Empty;
stream.CommitSnapshot();
return new Token(TokenType.Float, $"{leftPart}.{rightPart}{postfix}");
}
public override Token GetToken(TransactableTokenStream stream)
{
if (stream.Current != "/")
{
return null;
}
stream.TakeSnapshot();
stream.Consume();
string comment;
switch (stream.Current)
{
case "/":
stream.Consume();
comment = LexSingleLineComment(stream);
break;
case "*":
stream.Consume();
comment = LexMultiLineComment(stream);
break;
default:
stream.RollbackSnapshot();
return null;
}
stream.CommitSnapshot();
return new Token(TokenType.Comment, comment);
}
public override Token GetToken(TransactableTokenStream stream)
{
if (stream.Current != "\"")
{
return null;
}
stream.TakeSnapshot();
var tokenValue = stream.Current;
stream.Consume();
while (stream.Current != null && stream.Current != "\"")
{
tokenValue += stream.Current;
stream.Consume();
if (stream.Current == null)
{
stream.RollbackSnapshot();
return null;
}
}
tokenValue += stream.Current;
stream.Consume();
stream.CommitSnapshot();
return new Token(TokenType.String, tokenValue);
}
public bool IsMatch(TransactableTokenStream stream)
{
stream.TakeSnapshot();
var isMatch = GetToken(stream) != null;
stream.RollbackSnapshot();
return isMatch;
}
private static string GetPostfix(TransactableTokenStream stream)
{
if (stream.Current == "f" || stream.Current == "F")
{
var postfix = stream.Current;
stream.Consume();
return postfix;
}
return null;
}
protected static string LexInteger(TransactableTokenStream stream)
{
var number = string.Empty;
while (stream.Current != null && char.IsNumber(stream.Current[0]))
{
number += stream.Current;
stream.Consume();
}
return number;
}
private static string LexSingleLineComment(TransactableTokenStream stream)
{
var comment = string.Empty;
while (stream.Current != null && stream.Current != "\r" && stream.Peek(1) != "\n")
{
comment += stream.Current;
stream.Consume();
}
return $"//{comment}";
}
public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
var number = LexInteger(stream);
if (number == string.Empty)
{
stream.RollbackSnapshot();
return null;
}
return new Token(TokenType.Int, number);
}
private static string LexMultiLineComment(TransactableTokenStream stream)
{
var comment = string.Empty;
while (stream.Current != null && stream.Current != "*" && stream.Peek(1) != "/")
{
comment += stream.Current;
stream.Consume();
}
if (stream.Current == "*" && stream.Peek(1) == "/")
{
comment += stream.Current;
stream.Consume();
comment += stream.Current;
stream.Consume();
}
return $"/*{comment}";
}
public override Token GetToken(TransactableTokenStream stream)
{
stream.TakeSnapshot();
foreach (var character in _syntaxLiteral)
{
if (stream.Current == null || stream.Current != character.ToString())
{
stream.RollbackSnapshot();
return null;
}
stream.Consume();
}
stream.CommitSnapshot();
return new Token(_keyword, _syntaxLiteral);
}
public List<Token> Lex(string source)
{
var result = new List<Token>();
_tokenStream = new TransactableTokenStream(source);
var strategies = CreateLexerStrategies();
while (_tokenStream.Current != null)
{
var strategy = strategies.FirstOrDefault(x => x.IsMatch(_tokenStream));
if (strategy == null)
{
throw new KiwiSyntaxException($"Unexpected Token {_tokenStream.Current}");
}
result.Add(strategy.GetToken(_tokenStream));
}
return result;
}
public abstract Token GetToken(TransactableTokenStream stream);
