public static void Scan(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, ' ', '\t');
#endif
Source src = lexer.Src;
string value = string.Empty;
while (!src.ReachedEnd())
{
char current = src.Current;
if (!char.IsWhiteSpace(current))
{
src.Reverse();
break;
}
value += current;
src.Advance();
}
token.Type = TokenType.WhiteSpace;
token.Value = value;
}
public static bool ScanNewLine(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, '\r', '\n');
#endif
Source src = lexer.Src;
string value = string.Empty;
while (!src.ReachedEnd())
{
char current = src.Current;
if ((value != string.Empty) && (current != '\n'))
{
break;
}
value += current;
src.Advance();
}
src.Reverse();
token.Type = TokenType.NewLine;
token.Value = value;
return(true);
}
public void AdvancedTooMuchTest()
{
var src = new Source("abc");
src.Advance(10);
Assert.Equal(Source.InvalidChar, src.Current);
}
public void AdvanceTest()
{
var src = new Source("abc");
src.Advance();
Assert.Equal('b', src.Current);
}
public void reverseTest()
{
var src = new Source("abc");
src.Advance();
src.Reverse();
Assert.Equal('a', src.Current);
}
public static void Scan(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, chr => (chr == '_' || char.IsLetter(chr)),
"Character must be an underscore or letter.");
#endif
Source src = lexer.Src;
StringBuilder builder = lexer.Builder;
builder.Clear();
while (!src.ReachedEnd())
{
char current = src.Current;
if ((current == '_') || char.IsLetterOrDigit(current))
{
builder.Append(current);
src.Advance();
continue;
}
src.Reverse();
break;
}
string value = builder.ToString();
if (Keywords.TryGetTokenType(value, out TokenType tokenType))
{
token.Type = tokenType;
}
else
{
token.Type = TokenType.Identifier;
token.Value = value;
}
}
public static bool ScanMultiLine(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, '/');
Scanner.EnsureNext(lexer, '*');
#endif
Source src = lexer.Src;
StringBuilder builder = lexer.Builder;
builder.Clear();
bool inComment = true;
char previous = ' ';
while (!src.ReachedEnd())
{
char current = src.Current;
if ((builder.Length > "/*".Length) &&
(previous == '*') && (current == '/'))
{
inComment = false;
builder.Append(current);
break;
}
builder.Append(current);
src.Advance();
previous = current;
}
token.Type = TokenType.MultiComment;
token.Value = builder.ToString();
return(!inComment);
}
public static bool ScanSingleLine(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, '/');
Scanner.EnsureNext(lexer, '/');
#endif
Source src = lexer.Src;
StringBuilder builder = lexer.Builder;
builder.Clear();
while (!src.ReachedEnd())
{
char current = src.Current;
if (builder.Length >= "//".Length)
{
char next = src.Peek();
if ((next == '\r') || (next == '\n'))
{
builder.Append(current);
break;
}
}
builder.Append(current);
src.Advance();
}
token.Type = TokenType.SingleComment;
token.Value = builder.ToString();
return(true);
}
public static TokenType GetTokenType(Lexer lexer,
char nextExpected, TokenType then, TokenType @else)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
#endif
Source src = lexer.Src;
char next = src.Peek();
TokenType tokenType = (next == nextExpected)
? then
: @else;
if (next == nextExpected)
{
src.Advance();
}
return(tokenType);
}
public static bool ScanCharacter(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, '\'');
#endif
Source src = lexer.Src;
string result = string.Empty;
bool inCharacter = false;
bool escaped = false;
var illegalCharacters = new[] { '\r', '\n', '\t' };
var escapeCharacters = new[] { 'r', 'n', 't', '\'', '\\' };
token.Type = TokenType.Char;
while (!src.ReachedEnd())
{
char current = src.Current;
if ((current == '\'') && !escaped)
{
if (!inCharacter)
{
inCharacter = true;
}
else if (result == string.Empty)
{
return(false);
}
else
{
inCharacter = false;
break;
}
}
else if (escaped)
{
if (!escapeCharacters.Contains(current))
{
return(false);
}
result += current;
escaped = false;
}
else if (illegalCharacters.Contains(current))
{
return(false);
}
else if (result == string.Empty)
{
if (current == '\\')
{
escaped = true;
}
result += current;
}
else
{
return(false);
}
src.Advance();
}
token.Value = result;
return(!inCharacter && !escaped);
}
public static bool ScanString(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer, '"');
#endif
bool inString = false;
Source src = lexer.Src;
StringBuilder builder = lexer.Builder;
bool escaped = false;
var illegalCharacters = new[] { '\r', '\n' };
var escapeCharacters = new[] { 'r', 'n', 't', '"', '\\' };
builder.Clear();
token.Type = TokenType.String;
while (!src.ReachedEnd())
{
char current = src.Current;
if ((current == '"') && !escaped)
{
if (!inString)
{
inString = true;
}
else
{
inString = false;
break;
}
}
else if ((current == '\\') && !escaped)
{
escaped = true;
builder.Append(current);
}
else if (illegalCharacters.Contains(current))
{
return(false);
}
else
{
if (escaped && escapeCharacters.Contains(current))
{
escaped = false;
}
if (current != '\t')
{
builder.Append(current);
}
else
{
builder.Append('\\');
builder.Append('t');
}
}
src.Advance();
}
token.Type = TokenType.String;
token.Value = builder.ToString();
return(!inString);
}
public static bool Scan(Lexer lexer, ref Token token)
{
#if DEBUG
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
Scanner.EnsureCurrent(lexer,
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.');
#endif
Source src = lexer.Src;
string value = string.Empty;
bool dotEncountered = false;
bool success = true;
while (!src.ReachedEnd())
{
char current = src.Current;
char next = src.Peek();
if (dotEncountered && (next == '.'))
{
src.Advance();
success = false;
break;
}
if (current == '.')
{
dotEncountered = true;
if (!char.IsDigit(next))
{
success = false;
break;
}
}
else if (!char.IsDigit(current))
{
break;
}
value += current;
src.Advance();
}
if (success)
{
src.Reverse();
}
TokenType tokenType = dotEncountered
? TokenType.Float
: TokenType.Int;
token.Type = tokenType;
token.Value = value;
return(success);
}
