private static bool lexLineComment(LexBuffer source)
{
if (!source.tryRead("//"))
{
return(false);
}
source.readTillLineEnd();
return(true);
}
private static bool lexBlockComment(LexBuffer source)
{
if (!source.tryRead("/*"))
{
return(false);
}
do
{
source.readTillOneOf(new string[] { "/*", "*/" });
lexBlockComment(source);
} while (!source.tryRead("*/"));
return(true);
}
/// <summary>
/// Extracts all tokens from the provided buffer and returns a list of tokens.
/// </summary>
/// <param name="source"></param>
/// <returns></returns>
public List <Token> lexAll(LexBuffer source)
{
List <Token> tokens = new List <Token>();
Token?next;
while ((next = lexOne(source)).HasValue)
{
tokens.Add(next.Value);
}
return(tokens);
}
private static string lexName(LexBuffer source)
{
char first = source.nextChar();
if (!isLetter(first) && first != '_')
{
return(null);
}
return(source.readTill(c =>
!isLetter(c) &&
!isDigit(c) &&
c != '_'));
}
/// <summary>
/// Removes the next token in the buffer and returns it. If the end of the source is reached, returns null
/// </summary>
/// <param name="source">soure to extract a token from</param>
/// <returns>the next token or null if end of source</returns>
public Token?lexOne(LexBuffer source)
{
while (!source.atEnd())
{
source.consumeWhitespace();
if (source.atEnd())
{
return(null);
}
CodeLocation startLocation = source.GetLocation();
if (lexLineComment(source) || lexBlockComment(source))
{
continue;
}
string stringLiteral = lexStringLiteral(source);
if (stringLiteral != null)
{
return(new Token(TokenType.STRING, stringLiteral, startLocation, source.GetLocation()));
}
string name = lexName(source);
if (name != null)
{
return(new Token(TokenType.NAME, name, startLocation, source.GetLocation()));
}
string number = lexNumber(source);
if (number != null)
{
return(new Token(TokenType.NUMBER, number, startLocation, source.GetLocation()));
}
return(new Token(source.readChar(), startLocation, source.GetLocation()));
}
return(null);
}
private static string lexStringLiteral(LexBuffer source)
{
if (!source.tryRead("\""))
{
return(null);
}
string stringContents = "";
while (true)
{
stringContents += source.readTillOneOf(new string[] { "\\", "\"" });
if (source.tryRead("\""))
{
break;
}
else
{
source.consume("\\");
// TODO: handle other backslash character combos (like \n)
stringContents += source.readChar();
}
}
return(stringContents);
}