public static void Scan(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, ' ', '\t'); #endif Source src = lexer.Src; string value = string.Empty; while (!src.ReachedEnd()) { char current = src.Current; if (!char.IsWhiteSpace(current)) { src.Reverse(); break; } value += current; src.Advance(); } token.Type = TokenType.WhiteSpace; token.Value = value; }
public static bool ScanNewLine(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '\r', '\n'); #endif Source src = lexer.Src; string value = string.Empty; while (!src.ReachedEnd()) { char current = src.Current; if ((value != string.Empty) && (current != '\n')) { break; } value += current; src.Advance(); } src.Reverse(); token.Type = TokenType.NewLine; token.Value = value; return(true); }
public void AdvancedTooMuchTest() { var src = new Source("abc"); src.Advance(10); Assert.Equal(Source.InvalidChar, src.Current); }
public void AdvanceTest() { var src = new Source("abc"); src.Advance(); Assert.Equal('b', src.Current); }
public void reverseTest() { var src = new Source("abc"); src.Advance(); src.Reverse(); Assert.Equal('a', src.Current); }
public static void Scan(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, chr => (chr == '_' || char.IsLetter(chr)), "Character must be an underscore or letter."); #endif Source src = lexer.Src; StringBuilder builder = lexer.Builder; builder.Clear(); while (!src.ReachedEnd()) { char current = src.Current; if ((current == '_') || char.IsLetterOrDigit(current)) { builder.Append(current); src.Advance(); continue; } src.Reverse(); break; } string value = builder.ToString(); if (Keywords.TryGetTokenType(value, out TokenType tokenType)) { token.Type = tokenType; } else { token.Type = TokenType.Identifier; token.Value = value; } }
public static bool ScanMultiLine(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '/'); Scanner.EnsureNext(lexer, '*'); #endif Source src = lexer.Src; StringBuilder builder = lexer.Builder; builder.Clear(); bool inComment = true; char previous = ' '; while (!src.ReachedEnd()) { char current = src.Current; if ((builder.Length > "/*".Length) && (previous == '*') && (current == '/')) { inComment = false; builder.Append(current); break; } builder.Append(current); src.Advance(); previous = current; } token.Type = TokenType.MultiComment; token.Value = builder.ToString(); return(!inComment); }
public static bool ScanSingleLine(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '/'); Scanner.EnsureNext(lexer, '/'); #endif Source src = lexer.Src; StringBuilder builder = lexer.Builder; builder.Clear(); while (!src.ReachedEnd()) { char current = src.Current; if (builder.Length >= "//".Length) { char next = src.Peek(); if ((next == '\r') || (next == '\n')) { builder.Append(current); break; } } builder.Append(current); src.Advance(); } token.Type = TokenType.SingleComment; token.Value = builder.ToString(); return(true); }
public static TokenType GetTokenType(Lexer lexer, char nextExpected, TokenType then, TokenType @else) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } #endif Source src = lexer.Src; char next = src.Peek(); TokenType tokenType = (next == nextExpected) ? then : @else; if (next == nextExpected) { src.Advance(); } return(tokenType); }
public static bool ScanCharacter(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '\''); #endif Source src = lexer.Src; string result = string.Empty; bool inCharacter = false; bool escaped = false; var illegalCharacters = new[] { '\r', '\n', '\t' }; var escapeCharacters = new[] { 'r', 'n', 't', '\'', '\\' }; token.Type = TokenType.Char; while (!src.ReachedEnd()) { char current = src.Current; if ((current == '\'') && !escaped) { if (!inCharacter) { inCharacter = true; } else if (result == string.Empty) { return(false); } else { inCharacter = false; break; } } else if (escaped) { if (!escapeCharacters.Contains(current)) { return(false); } result += current; escaped = false; } else if (illegalCharacters.Contains(current)) { return(false); } else if (result == string.Empty) { if (current == '\\') { escaped = true; } result += current; } else { return(false); } src.Advance(); } token.Value = result; return(!inCharacter && !escaped); }
public static bool ScanString(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '"'); #endif bool inString = false; Source src = lexer.Src; StringBuilder builder = lexer.Builder; bool escaped = false; var illegalCharacters = new[] { '\r', '\n' }; var escapeCharacters = new[] { 'r', 'n', 't', '"', '\\' }; builder.Clear(); token.Type = TokenType.String; while (!src.ReachedEnd()) { char current = src.Current; if ((current == '"') && !escaped) { if (!inString) { inString = true; } else { inString = false; break; } } else if ((current == '\\') && !escaped) { escaped = true; builder.Append(current); } else if (illegalCharacters.Contains(current)) { return(false); } else { if (escaped && escapeCharacters.Contains(current)) { escaped = false; } if (current != '\t') { builder.Append(current); } else { builder.Append('\\'); builder.Append('t'); } } src.Advance(); } token.Type = TokenType.String; token.Value = builder.ToString(); return(!inString); }
public static bool Scan(Lexer lexer, ref Token token) { #if DEBUG if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } Scanner.EnsureCurrent(lexer, '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.'); #endif Source src = lexer.Src; string value = string.Empty; bool dotEncountered = false; bool success = true; while (!src.ReachedEnd()) { char current = src.Current; char next = src.Peek(); if (dotEncountered && (next == '.')) { src.Advance(); success = false; break; } if (current == '.') { dotEncountered = true; if (!char.IsDigit(next)) { success = false; break; } } else if (!char.IsDigit(current)) { break; } value += current; src.Advance(); } if (success) { src.Reverse(); } TokenType tokenType = dotEncountered ? TokenType.Float : TokenType.Int; token.Type = tokenType; token.Value = value; return(success); }