static TokenData ScanSingleChar(TextInputRange input, Token token) { input.Extend(); return(new TokenData { Range = input.Clone(), Type = token }); }
static TokenData ScanNewLine(TextInputRange input) { input.NewLine(); input.ExtendWhitespaces(); return(new TokenData { Range = input.Clone(), Type = Token.NewLineIndentation }); }
static TokenData ScanWhitespaces(TextInputRange input) { input.Extend(); input.ExtendWhitespaces(); return(new TokenData { Range = input.Clone(), Type = Token.WhiteSpaceSeperator }); }
static TokenData ScanComment(TextInputRange input) { var comment = CommentScanner.Scan(input); Debug.Assert(comment); return(new TokenData { Range = input.Clone(), Type = Token.Comment }); }
// Scan basic double quoted strings public static IStringLiteral Scan(TextInputRange input) { var chr = input.EndChar; if (!IsDoubleQuote(chr)) { return(null); } input.Extend(); var result = new StringBuilder(); while (true) { chr = input.EndChar; if (chr == '\0') { return(null); // file end or invalid input } if (IsDoubleQuote(chr)) { break; } if (input.IsEndNewline) { input.NewLine(); } else { if (!IsTab(chr) && char.IsControl(chr)) { HandleControl(input); } else { if (IsBackslash(chr)) { if (!HandleEscape(input, result)) { return(null); } } else { result.Append(chr); input.Extend(); } } } } input.Extend(); return(new StringLiteral { Content = result.ToString(), Range = input.Clone() }); }
// this will scan the longest valid regular entry public static IIdentifierLiteral Scan(TextInputRange input) { if (!IsStart(input)) { input.Backtrack(); // dot might have been skipped return(null); } do { input.Extend(); } while (IsContinue(input)); return(new IdentifierLiteral { Content = input.Text, Range = input.Clone() }); }
public static IEnumerable <TokenData> ScanFile(TextFile file) { var input = new TextInputRange { File = file }; while (true) { input.Collapse(); var chr = input.EndChar; // ReSharper disable once SwitchStatementMissingSomeCases switch (chr) { case '\0': yield break; case ' ': case '\t': yield return(ScanWhitespaces(input)); continue; case '\n': case '\r': yield return(ScanNewLine(input)); continue; case '#': yield return(ScanComment(input)); continue; case ',': yield return(ScanSingleChar(input, Token.CommaSeparator)); continue; case ';': yield return(ScanSingleChar(input, Token.SemicolonSeparator)); continue; case '[': yield return(ScanSingleChar(input, Token.SquareBracketOpen)); continue; case ']': yield return(ScanSingleChar(input, Token.SquareBracketClose)); continue; case '(': yield return(ScanSingleChar(input, Token.BracketOpen)); continue; case ')': yield return(ScanSingleChar(input, Token.BracketClose)); continue; case '"': yield return(ScanStringLiteral(input)); continue; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': yield return(ScanNumberLiteral(input)); continue; } var identifierLiteral = IdentifierScanner.Scan(input); if (identifierLiteral != null) { yield return(new TokenData { Range = identifierLiteral.Range, Type = Token.IdentifierLiteral, Data = identifierLiteral }); continue; } var operatorLiteral = OperatorScanner.Scan(input); if (operatorLiteral != null) { yield return(new TokenData { Range = operatorLiteral.Range, Type = Token.OperatorLiteral, Data = operatorLiteral }); continue; } input.Extend(); yield return(new TokenData { Range = input.Clone(), Type = Token.InvalidCharacter }); } }