public static IEnumerable <TokenData> ScanFile(TextFile file) { var input = new TextInputRange { File = file }; while (true) { input.Collapse(); var chr = input.EndChar; // ReSharper disable once SwitchStatementMissingSomeCases switch (chr) { case '\0': yield break; case ' ': case '\t': yield return(ScanWhitespaces(input)); continue; case '\n': case '\r': yield return(ScanNewLine(input)); continue; case '#': yield return(ScanComment(input)); continue; case ',': yield return(ScanSingleChar(input, Token.CommaSeparator)); continue; case ';': yield return(ScanSingleChar(input, Token.SemicolonSeparator)); continue; case '[': yield return(ScanSingleChar(input, Token.SquareBracketOpen)); continue; case ']': yield return(ScanSingleChar(input, Token.SquareBracketClose)); continue; case '(': yield return(ScanSingleChar(input, Token.BracketOpen)); continue; case ')': yield return(ScanSingleChar(input, Token.BracketClose)); continue; case '"': yield return(ScanStringLiteral(input)); continue; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': yield return(ScanNumberLiteral(input)); continue; } var identifierLiteral = IdentifierScanner.Scan(input); if (identifierLiteral != null) { yield return(new TokenData { Range = identifierLiteral.Range, Type = Token.IdentifierLiteral, Data = identifierLiteral }); continue; } var operatorLiteral = OperatorScanner.Scan(input); if (operatorLiteral != null) { yield return(new TokenData { Range = operatorLiteral.Range, Type = Token.OperatorLiteral, Data = operatorLiteral }); continue; } input.Extend(); yield return(new TokenData { Range = input.Clone(), Type = Token.InvalidCharacter }); } }