protected override Token IsMatchImpl(Tokenizer tokenizer)
{
var leftOperand = GetIntegers(tokenizer);
if (leftOperand != null)
{
bool isFloat = false;
string numberValue = leftOperand;
if (tokenizer.Current == ".")
{
tokenizer.Consume();
var rightOperand = GetIntegers(tokenizer);
if (rightOperand != null)
{
numberValue += "." + rightOperand;
isFloat = true;
}
}
if (tokenizer.Current == "e")
{
string rightOperand = "";
tokenizer.Consume();
if (tokenizer.Current == "-")
{
rightOperand = "-";
tokenizer.Consume();
}
rightOperand += GetIntegers(tokenizer);
if (rightOperand != null)
{
numberValue += "e" + rightOperand;
isFloat = true;
}
}
if (!tokenizer.End())
{
if (Char.IsLetter(tokenizer.Current, 0))
{
return(null);
}
}
if (isFloat)
{
return(new Token(TokenType.FloatValue, numberValue));
}
return(new Token(TokenType.IntValue, leftOperand));
}
return(null);
}
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item1).IsMatch(tokenizer) == null)
{
return(null);
}
else
{
tokenizer.Consume();
while (!tokenizer.End() && new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null)
{
tokenizer.Consume();
}
if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null)
{
tokenizer.Consume();
}
TokenType tokenType = TokenType.OpenBlockComment;
if (CommentDelim == LINE_COMMENT)
{
tokenType = TokenType.OpenLineComment;
}
return(new Token(tokenType, CommentDelim.Item1));
}
}
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
foreach (var character in Match)
{
if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture))
{
tokenizer.Consume();
}
else
{
return(null);
}
}
bool found;
if (!AllowAsSubString)
{
var next = tokenizer.Current;
found = String.IsNullOrWhiteSpace(next) || SpecialCharacters.Any(character => character.Match == next);
}
else
{
found = true;
}
return(found ? new Token(TokenType, Match) : null);
}
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
var str = new StringBuilder();
if (tokenizer.Current == StringDelim)
{
tokenizer.Consume();
while (!tokenizer.End() && tokenizer.Current != StringDelim)
{
str.Append(tokenizer.Current);
tokenizer.Consume();
}
if (tokenizer.End())
{
return(null);
}
if (tokenizer.Current == StringDelim)
{
tokenizer.Consume();
}
}
TokenType tokenType;
if (StringDelim == TIC)
{
tokenType = TokenType.CharValue;
if (str.Length > 1)
{
return(null);
}
}
else
{
tokenType = TokenType.StringValue;
}
return(str.Length > 0 ? new Token(tokenType, StringDelim + str + StringDelim) : null);
}
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
bool foundWhiteSpace = false;
while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current))
{
foundWhiteSpace = true;
tokenizer.Consume();
}
return(foundWhiteSpace ? new Token(TokenType.WhiteSpace) : null);
}
private String GetIntegers(Tokenizer tokenizer)
{
var regex = new Regex("[0-9]");
String num = null;
while (tokenizer.Current != null && regex.IsMatch(tokenizer.Current))
{
num += tokenizer.Current;
tokenizer.Consume();
}
return(num != null ? num : null);
}
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
String current = null;
while (!tokenizer.End() && !String.IsNullOrWhiteSpace(tokenizer.Current) && SpecialCharacters.All(m => m.Match != tokenizer.Current))
{
current += tokenizer.Current;
tokenizer.Consume();
}
TokenType tokenType = TokenType.Unknown;
if (current == null)
{
return(null);
}
if (Char.IsLetter(current, 0))
{
tokenType = TokenType.Identifier;
}
return(new Token(tokenType, current));
}