private static readonly char[] longCommentID2 = { '-', '[', '=' }; //TODO: flawed approach? what if --[=asdfadf]?
public static List <Token> Tokenize(TextReader textReader)
{
Validation.Requires.NotNull(textReader, nameof(textReader));
TrackableTextReader trackableTextReader = new TrackableTextReader(textReader);
Token nextToken;
List <Trivia> trivia;
List <Token> tokenList = new List <Token>();
while (!trackableTextReader.EndOfStream())
{
int fullStart = (int)trackableTextReader.Position;
trivia = ConsumeTrivia(trackableTextReader);
nextToken = ReadNextToken(trackableTextReader, trivia, fullStart);
tokenList.Add(nextToken);
if (trackableTextReader.EndOfStream() && nextToken.Kind != SyntaxKind.EndOfFile)
{
nextToken = new Token(SyntaxKind.EndOfFile, string.Empty, new List <Trivia>(), nextToken.End, nextToken.End);
tokenList.Add(nextToken);
}
}
if (tokenList.Count == 0)
{
//If there is an empty program send back an end of file token.
tokenList.Add(new Token(SyntaxKind.EndOfFile, "", new List <Trivia>(), 0, 0));
}
return(tokenList);
}
private static Token ReadNextToken(TrackableTextReader stream, List <Trivia> trivia, int fullStart)
{
char nextChar;
if (stream.EndOfStream())
{
return(new Token(SyntaxKind.EndOfFile, "", trivia, fullStart, (int)stream.Position));
}
nextChar = stream.Peek();
// Keyword or Identifier
if (char.IsLetter(nextChar) || (nextChar == '_'))
{
return(ReadAlphaToken(stream, trivia, fullStart));
}
// Number
else if (char.IsDigit(nextChar))
{
return(ReadNumberToken(stream, trivia, fullStart));
}
// String
else if (IsQuote(nextChar))
{
return(ReadStringToken(nextChar, stream, trivia, fullStart));
}
// Punctuation Bracket Operator
else
{
return(ReadSymbolToken(stream, trivia, fullStart));
}
}
private static Trivia ReadLongComment(TrackableTextReader stream, string commentSoFar, int?level)
{
if (level == null)
{
throw new ArgumentNullException(nameof(level));
}
//TODO: re-write without regex
Regex closeBracketPattern = new Regex(@"\]={" + level.ToString() + @"}\]");
while (!closeBracketPattern.IsMatch(commentSoFar) && !stream.EndOfStream())
{
commentSoFar += stream.ReadChar();
}
return(new Trivia(SyntaxKind.Comment, commentSoFar));
}
public static int CountLevels(bool validateCount, char character, int counter, TrackableTextReader stream, StringBuilder builder)
{
int levelCount = counter;
while (character == '=' && !stream.EndOfStream())
{
builder.Append(stream.ReadChar());
if (validateCount)
{
levelCount--;
}
else
{
levelCount++;
}
character = stream.Peek();
}
return(levelCount);
}
private static Token ReadNumberToken(TrackableTextReader stream, List <Trivia> trivia, int fullStart)
{
StringBuilder number = new StringBuilder();
int tokenStartPosition = (int)stream.Position;
char next = stream.Peek();
// TODO: verify only one decimal point
while (IsValidNumber(next))
{
number.Append(stream.ReadChar());
next = stream.Peek();
}
if (IsValidTerminator(next) || stream.EndOfStream())
{
return(new Token(SyntaxKind.Number, number.ToString(), trivia, fullStart, tokenStartPosition));
}
else
{
return(new Token(SyntaxKind.Unknown, number.ToString(), trivia, fullStart, tokenStartPosition)); //TODO: Deal with invalid number/identifier: "234kjs"
}
}
private static Token ReadStringToken(char stringDelimiter, TrackableTextReader stream, List <Trivia> leadingTrivia, int fullStart)
{
StringBuilder fullString = new StringBuilder();
int tokenStartPosition = (int)stream.Position;
SyntaxKind type = SyntaxKind.String;
char nextChar;
switch (stringDelimiter)
{
case '"':
case '\'':
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
bool terminateString = false;
while ((nextChar != stringDelimiter) && !stream.EndOfStream() && !terminateString)
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
if (nextChar == '\r' || nextChar == '\n')
{
type = SyntaxKind.UnterminatedString;
terminateString = true;
}
}
if (nextChar == stringDelimiter || terminateString)
{
fullString.Append(stream.ReadChar());
return(new Token(type, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition));
}
else
{
return(new Token(SyntaxKind.EndOfFile, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition)); //TODO bug... should return a string then a EOF token right?
}
case '[':
fullString.Append(stream.ReadChar());
int bracketLevel = 0;
nextChar = stream.Peek();
bracketLevel = CountLevels(false, nextChar, 0, stream, fullString);
nextChar = stream.Peek();
if (nextChar == '[')
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
//Lua ignores a new line directly after the opening delimiter of a string.
if (nextChar == '\r' || nextChar == '\n')
{
if (nextChar == '\r')
{
stream.ReadChar();
}
if (stream.Peek() == '\n')
{
stream.ReadChar();
}
type = SyntaxKind.IgnoreNewLineString;
}
while (!stream.EndOfStream())
{
if (nextChar == ']')
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
int currentLevel = bracketLevel;
currentLevel = CountLevels(true, nextChar, currentLevel, stream, fullString);
nextChar = stream.Peek();
if ((nextChar == ']') && (currentLevel == 0))
{
fullString.Append(stream.ReadChar());
return(new Token(type, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition));
}
}
else
{
fullString.Append(stream.ReadChar());
}
nextChar = stream.Peek();
}
return(new Token(SyntaxKind.UnterminatedString, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition));
}
else
{
if (bracketLevel == 0)
{
return(new Token(SyntaxKind.OpenBracket, nextChar.ToString(), leadingTrivia, fullStart, tokenStartPosition));
}
else
{
// Error, not valid syntax
return(new Token(SyntaxKind.Unknown, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition));
}
}
default:
throw new ArgumentOutOfRangeException(nameof(stringDelimiter), "Unrecognized String delimiter");
}
}
private static readonly char[] longCommentID2 = { '-', '[', '=' }; //TODO: flawed approach? what if --[=asdfadf]?
public static List<Token> Tokenize(TextReader textReader)
{
Validation.Requires.NotNull(textReader, nameof(textReader));
TrackableTextReader trackableTextReader = new TrackableTextReader(textReader);
Token nextToken;
List<Trivia> trivia;
List<Token> tokenList = new List<Token>();
while (!trackableTextReader.EndOfStream())
{
int fullStart = (int)trackableTextReader.Position;
trivia = ConsumeTrivia(trackableTextReader);
nextToken = ReadNextToken(trackableTextReader, trivia, fullStart);
tokenList.Add(nextToken);
if (trackableTextReader.EndOfStream() && nextToken.Kind != SyntaxKind.EndOfFile)
{
nextToken = new Token(SyntaxKind.EndOfFile, string.Empty, new List<Trivia>(), nextToken.End, nextToken.End);
tokenList.Add(nextToken);
}
}
if (tokenList.Count == 0)
{
//If there is an empty program send back an end of file token.
tokenList.Add(new Token(SyntaxKind.EndOfFile, "", new List<Trivia>(), 0, 0));
}
return tokenList;
}
public static int CountLevels(bool validateCount, char character, int counter, TrackableTextReader stream, StringBuilder builder)
{
int levelCount = counter;
while (character == '=' && !stream.EndOfStream())
{
builder.Append(stream.ReadChar());
if (validateCount)
{
levelCount--;
}
else
{
levelCount++;
}
character = stream.Peek();
}
return levelCount;
}
private static Token ReadStringToken(char stringDelimiter, TrackableTextReader stream, List<Trivia> leadingTrivia, int fullStart)
{
StringBuilder fullString = new StringBuilder();
int tokenStartPosition = (int)stream.Position;
SyntaxKind type = SyntaxKind.String;
char nextChar;
switch (stringDelimiter)
{
case '"':
case '\'':
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
bool terminateString = false;
while ((nextChar != stringDelimiter) && !stream.EndOfStream() && !terminateString)
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
if (nextChar == '\r' || nextChar == '\n')
{
type = SyntaxKind.UnterminatedString;
terminateString = true;
}
}
if (nextChar == stringDelimiter || terminateString)
{
fullString.Append(stream.ReadChar());
return new Token(type, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition);
}
else
{
return new Token(SyntaxKind.EndOfFile, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition); //TODO bug... should return a string then a EOF token right?
}
case '[':
fullString.Append(stream.ReadChar());
int bracketLevel = 0;
nextChar = stream.Peek();
bracketLevel = CountLevels(false, nextChar, 0, stream, fullString);
nextChar = stream.Peek();
if (nextChar == '[')
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
//Lua ignores a new line directly after the opening delimiter of a string.
if (nextChar == '\r' || nextChar == '\n')
{
if (nextChar == '\r')
stream.ReadChar();
if (stream.Peek() == '\n')
stream.ReadChar();
type = SyntaxKind.IgnoreNewLineString;
}
while (!stream.EndOfStream())
{
if (nextChar == ']')
{
fullString.Append(stream.ReadChar());
nextChar = stream.Peek();
int currentLevel = bracketLevel;
currentLevel = CountLevels(true, nextChar, currentLevel, stream, fullString);
nextChar = stream.Peek();
if ((nextChar == ']') && (currentLevel == 0))
{
fullString.Append(stream.ReadChar());
return new Token(type, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition);
}
}
else
{
fullString.Append(stream.ReadChar());
}
nextChar = stream.Peek();
}
return new Token(SyntaxKind.UnterminatedString, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition);
}
else
{
if (bracketLevel == 0)
{
return new Token(SyntaxKind.OpenBracket, nextChar.ToString(), leadingTrivia, fullStart, tokenStartPosition);
}
else
{
// Error, not valid syntax
return new Token(SyntaxKind.Unknown, fullString.ToString(), leadingTrivia, fullStart, tokenStartPosition);
}
}
default:
throw new ArgumentOutOfRangeException(nameof(stringDelimiter), "Unrecognized String delimiter");
}
}
private static Token ReadNumberToken(TrackableTextReader stream, List<Trivia> trivia, int fullStart)
{
StringBuilder number = new StringBuilder();
int tokenStartPosition = (int)stream.Position;
char next = stream.Peek();
// TODO: verify only one decimal point
while (IsValidNumber(next))
{
number.Append(stream.ReadChar());
next = stream.Peek();
}
if (IsValidTerminator(next) || stream.EndOfStream())
{
return new Token(SyntaxKind.Number, number.ToString(), trivia, fullStart, tokenStartPosition);
}
else
{
return new Token(SyntaxKind.Unknown, number.ToString(), trivia, fullStart, tokenStartPosition); //TODO: Deal with invalid number/identifier: "234kjs"
}
}
private static Token ReadNextToken(TrackableTextReader stream, List<Trivia> trivia, int fullStart)
{
char nextChar;
if (stream.EndOfStream())
{
return new Token(SyntaxKind.EndOfFile, "", trivia, fullStart, (int)stream.Position);
}
nextChar = stream.Peek();
// Keyword or Identifier
if (char.IsLetter(nextChar) || (nextChar == '_'))
{
return ReadAlphaToken(stream, trivia, fullStart);
}
// Number
else if (char.IsDigit(nextChar))
{
return ReadNumberToken(stream, trivia, fullStart);
}
// String
else if (IsQuote(nextChar))
{
return ReadStringToken(nextChar, stream, trivia, fullStart);
}
// Punctuation Bracket Operator
else
{
return ReadSymbolToken(stream, trivia, fullStart);
}
}
private static Trivia ReadLongComment(TrackableTextReader stream, string commentSoFar, int? level)
{
if (level == null)
{
throw new ArgumentNullException(nameof(level));
}
//TODO: re-write without regex
Regex closeBracketPattern = new Regex(@"\]={" + level.ToString() + @"}\]");
while (!closeBracketPattern.IsMatch(commentSoFar) && !stream.EndOfStream())
{
commentSoFar += stream.ReadChar();
}
return new Trivia(SyntaxKind.Comment, commentSoFar);
}
