private static void SaveCurrentCharToNewToken(TokenizationState state, SqlTokenType tokenType) { char charToSave = state.CurrentChar; state.HasUnprocessedCurrentCharacter = false; //because we're using it now! SaveToken(state, tokenType, charToSave.ToString()); }
private static TextParser <TokenType> GetParserToTry(TokenizationState <TokenType> state) { switch (state?.Previous?.Kind) { case null: return(StartOfFileParser); case TokenType.KeyChars: case TokenType.KeyEscapeSequence: case TokenType.KeyPhysicalNewLine: return(AfterKeyComponentParser); case TokenType.Separator: return(AfterSeparatorParser); case TokenType.Value: return(AfterValueComponentParser); case TokenType.Comment: return(AfterCommentParser); case TokenType.Whitespace: return(AfterWhitespaceParser); default: throw new ArgumentOutOfRangeException(); } }
/// <summary> /// Tokenize <paramref name="source"/>. /// </summary> /// <param name="source">The source to tokenize.</param> /// <returns>A result with the list of tokens or an error.</returns> /// <exception cref="ArgumentNullException"><paramref name="source"/> is null.</exception> /// <exception cref="ParseException">The tokenizer could not correctly perform tokenization.</exception> public Result<TokenList<TKind>> TryTokenize(string source) { if (source == null) throw new ArgumentNullException(nameof(source)); var state = new TokenizationState<TKind>(); var sourceSpan = new TextSpan(source); var remainder = sourceSpan; var results = new List<Token<TKind>>(); foreach (var result in Tokenize(sourceSpan, state)) { if (!result.HasValue) return Result.CastEmpty<TKind, TokenList<TKind>>(result); if (result.Remainder == remainder) // Broken parser, not a failed parsing. throw new ParseException($"Zero-width tokens are not supported; token {Presentation.FormatExpectation(result.Value)} at position {result.Location.Position}."); remainder = result.Remainder; var token = new Token<TKind>(result.Value, result.Location.Until(result.Remainder)); state.Previous = token; results.Add(token); } var value = new TokenList<TKind>(results.ToArray()); return Result.Value(value, sourceSpan, remainder); }
protected override IEnumerable <Result <int> > Tokenize(TextSpan span, TokenizationState <int> state) { Assert.NotNull(state); Assert.Null(state.Previous); var next = span.ConsumeChar(); yield return(Result.Value(0, next.Location, next.Remainder)); for (var i = 1; i < span.Length; ++i) { Assert.NotNull(state.Previous); Assert.Equal(i - 1, state.Previous !.Value.Kind); next = next.Remainder.ConsumeChar(); yield return(Result.Value(i, next.Location, next.Remainder)); } }
private static void SaveToken(TokenizationState state, SqlTokenType tokenType, string tokenValue) { var foundToken = new Token(tokenType, tokenValue); state.TokenContainer.Add(foundToken); long positionOfLastCharacterInToken = state.InputReader.LastCharacterPosition - (state.HasUnprocessedCurrentCharacter ? 1 : 0); if (state.RequestedMarkerPosition != null && state.TokenContainer.MarkerToken == null && state.RequestedMarkerPosition <= positionOfLastCharacterInToken ) { state.TokenContainer.MarkerToken = foundToken; //TODO: this is wrong for container types, as commented elsewhere. the marker position will be too high. var rawPositionInToken = foundToken.Value.Length - (positionOfLastCharacterInToken - state.RequestedMarkerPosition); // temporarily bypass overflow issues without fixing underlying problem state.TokenContainer.MarkerPosition = rawPositionInToken > foundToken.Value.Length ? foundToken.Value.Length : rawPositionInToken; } }
protected override IEnumerable <Result <SqlToken> > Tokenize( TextSpan stringSpan, TokenizationState <SqlToken> tokenizationState) { var next = SkipWhiteSpace(stringSpan); if (!next.HasValue) { yield break; } do { if (char.IsDigit(next.Value)) { var result = OneOf( (SqlToken.HexNumber, SqlParser.HexInteger), (SqlToken.Number, SqlParser.Real) )(next); yield return(result.parseResult); next = result.next; } else if (char.IsLetter(next.Value) || next.Value == '_') { var beginIdentifier = next.Location; do { next = next.Remainder.ConsumeChar(); }while (next.HasValue && (char.IsLetterOrDigit(next.Value) || next.Value == '_')); if (TryGetKeyword(beginIdentifier.Until(next.Location), out var keyword)) { yield return(Result.Value(keyword, beginIdentifier, next.Location)); } else { yield return(Result.Value(SqlToken.Identifier, beginIdentifier, next.Location)); } } else { var beginSymbol = next.Location; do { next = next.Remainder.ConsumeChar(); }while (next.HasValue && !IsIdent(next)); var symbolText = beginSymbol.Until(next.Location).ToStringValue().TrimSpaces(); if (TryGetSymbol(symbolText, out var symbol)) { yield return(Result.Value(symbol, beginSymbol, next.Location)); } else { yield return(Result.Value(SqlToken.Unknown, beginSymbol, next.Location)); } } next = SkipWhiteSpace(next.Location); } while (next.HasValue); }
private static void CompleteToken(TokenizationState state, bool nextCharRead) { if (state.CurrentTokenizationType == null) { throw new Exception("Cannot complete Token, as there is no current Tokenization Type"); } switch (state.CurrentTokenizationType) { case SqlTokenizationType.BlockComment: SaveToken(state, SqlTokenType.MultiLineComment, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.OtherNode: SaveToken(state, SqlTokenType.OtherNode, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.PseudoName: SaveToken(state, SqlTokenType.PseudoName, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleLineComment: SaveToken(state, SqlTokenType.SingleLineComment, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleLineCommentCStyle: SaveToken(state, SqlTokenType.SingleLineCommentCStyle, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleHyphen: SaveToken(state, SqlTokenType.OtherOperator, "-"); break; case SqlTokenizationType.SingleDollar: SaveToken(state, SqlTokenType.MonetaryValue, "$"); break; case SqlTokenizationType.SingleSlash: SaveToken(state, SqlTokenType.OtherOperator, "/"); break; case SqlTokenizationType.WhiteSpace: SaveToken(state, SqlTokenType.WhiteSpace, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleN: SaveToken(state, SqlTokenType.OtherNode, "N"); break; case SqlTokenizationType.SingleExclamation: SaveToken(state, SqlTokenType.OtherNode, "!"); break; case SqlTokenizationType.SinglePipe: SaveToken(state, SqlTokenType.OtherNode, "|"); break; case SqlTokenizationType.SingleGT: SaveToken(state, SqlTokenType.OtherOperator, ">"); break; case SqlTokenizationType.SingleLT: SaveToken(state, SqlTokenType.OtherOperator, "<"); break; case SqlTokenizationType.NString: SaveToken(state, SqlTokenType.NationalString, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.String: SaveToken(state, SqlTokenType.String, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.QuotedString: SaveToken(state, SqlTokenType.QuotedString, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.BracketQuotedName: SaveToken(state, SqlTokenType.BracketQuotedName, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.OtherOperator: case SqlTokenizationType.SingleOtherCompoundableOperator: SaveToken(state, SqlTokenType.OtherOperator, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleZero: SaveToken(state, SqlTokenType.Number, "0"); break; case SqlTokenizationType.SinglePeriod: SaveToken(state, SqlTokenType.Period, "."); break; case SqlTokenizationType.SingleAsterisk: SaveToken(state, SqlTokenType.Asterisk, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.SingleEquals: SaveToken(state, SqlTokenType.EqualsSign, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.Number: case SqlTokenizationType.DecimalValue: case SqlTokenizationType.FloatValue: SaveToken(state, SqlTokenType.Number, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.BinaryValue: SaveToken(state, SqlTokenType.BinaryValue, state.CurrentTokenValue.ToString()); break; case SqlTokenizationType.MonetaryValue: SaveToken(state, SqlTokenType.MonetaryValue, state.CurrentTokenValue.ToString()); break; default: throw new Exception("Unrecognized SQL Node Type"); } state.CurrentTokenizationType = null; }
private static void ProcessOrOpenToken(TokenizationState state) { if (state.CurrentTokenizationType != null) { throw new Exception("Cannot start a new Token: existing Tokenization Type is not null"); } if (!state.HasUnprocessedCurrentCharacter) { throw new Exception("Cannot start a new Token: no (outstanding) current character specified!"); } //start a new value. state.CurrentTokenValue.Length = 0; if (IsWhitespace(state.CurrentChar)) { state.CurrentTokenizationType = SqlTokenizationType.WhiteSpace; state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == '-') { state.CurrentTokenizationType = SqlTokenizationType.SingleHyphen; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '$') { state.CurrentTokenizationType = SqlTokenizationType.SingleDollar; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '/') { state.CurrentTokenizationType = SqlTokenizationType.SingleSlash; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == 'N') { state.CurrentTokenizationType = SqlTokenizationType.SingleN; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later except N-string case } else if (state.CurrentChar == '\'') { state.CurrentTokenizationType = SqlTokenizationType.String; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing } else if (state.CurrentChar == '"') { state.CurrentTokenizationType = SqlTokenizationType.QuotedString; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing } else if (state.CurrentChar == '[') { state.CurrentTokenizationType = SqlTokenizationType.BracketQuotedName; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing } else if (state.CurrentChar == '(') { SaveCurrentCharToNewToken(state, SqlTokenType.OpenParens); } else if (state.CurrentChar == ')') { SaveCurrentCharToNewToken(state, SqlTokenType.CloseParens); } else if (state.CurrentChar == ',') { SaveCurrentCharToNewToken(state, SqlTokenType.Comma); } else if (state.CurrentChar == '.') { state.CurrentTokenizationType = SqlTokenizationType.SinglePeriod; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '0') { state.CurrentTokenizationType = SqlTokenizationType.SingleZero; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar >= '1' && state.CurrentChar <= '9') { state.CurrentTokenizationType = SqlTokenizationType.Number; state.ConsumeCurrentCharacterIntoToken(); } else if (IsCurrencyPrefix(state.CurrentChar)) { state.CurrentTokenizationType = SqlTokenizationType.MonetaryValue; state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == ';') { SaveCurrentCharToNewToken(state, SqlTokenType.Semicolon); } else if (state.CurrentChar == ':') { SaveCurrentCharToNewToken(state, SqlTokenType.Colon); } else if (state.CurrentChar == '*') { state.CurrentTokenizationType = SqlTokenizationType.SingleAsterisk; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '=') { state.CurrentTokenizationType = SqlTokenizationType.SingleEquals; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '<') { state.CurrentTokenizationType = SqlTokenizationType.SingleLT; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '>') { state.CurrentTokenizationType = SqlTokenizationType.SingleGT; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '!') { state.CurrentTokenizationType = SqlTokenizationType.SingleExclamation; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (state.CurrentChar == '|') { state.CurrentTokenizationType = SqlTokenizationType.SinglePipe; state.HasUnprocessedCurrentCharacter = false; //purposefully swallowing, will be reinserted later } else if (IsCompoundableOperatorCharacter(state.CurrentChar)) { state.CurrentTokenizationType = SqlTokenizationType.SingleOtherCompoundableOperator; state.ConsumeCurrentCharacterIntoToken(); } else if (IsOperatorCharacter(state.CurrentChar)) { SaveCurrentCharToNewToken(state, SqlTokenType.OtherOperator); } else { state.CurrentTokenizationType = SqlTokenizationType.OtherNode; state.ConsumeCurrentCharacterIntoToken(); } }
private static void SwallowOutstandingCharacterAndCompleteToken(TokenizationState state) { //this is for cases where we *know* we are swallowing the "current character" (not putting it in the output) state.HasUnprocessedCurrentCharacter = false; CompleteToken(state, false); }
private static void AppendCharAndCompleteToken(TokenizationState state) { state.ConsumeCurrentCharacterIntoToken(); CompleteToken(state, false); }
private static void CompleteTokenAndProcessNext(TokenizationState state) { CompleteToken(state, true); ProcessOrOpenToken(state); }
public ITokenList TokenizeSQL(string inputSQL, long?requestedMarkerPosition) { var state = new TokenizationState(inputSQL, requestedMarkerPosition); state.ReadNextCharacter(); while (state.HasUnprocessedCurrentCharacter) { if (state.CurrentTokenizationType == null) { ProcessOrOpenToken(state); state.ReadNextCharacter(); continue; } switch (state.CurrentTokenizationType.Value) { case SqlTokenizationType.WhiteSpace: if (IsWhitespace(state.CurrentChar)) { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SinglePeriod: if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.CurrentTokenizationType = SqlTokenizationType.DecimalValue; state.CurrentTokenValue.Append('.'); state.ConsumeCurrentCharacterIntoToken(); } else { state.CurrentTokenValue.Append('.'); CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleZero: if (state.CurrentChar == 'x' || state.CurrentChar == 'X') { state.CurrentTokenizationType = SqlTokenizationType.BinaryValue; state.CurrentTokenValue.Append('0'); state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.CurrentTokenizationType = SqlTokenizationType.Number; state.CurrentTokenValue.Append('0'); state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == '.') { state.CurrentTokenizationType = SqlTokenizationType.DecimalValue; state.CurrentTokenValue.Append('0'); state.ConsumeCurrentCharacterIntoToken(); } else { state.CurrentTokenValue.Append('0'); CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.Number: if (state.CurrentChar == 'e' || state.CurrentChar == 'E') { state.CurrentTokenizationType = SqlTokenizationType.FloatValue; state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == '.') { state.CurrentTokenizationType = SqlTokenizationType.DecimalValue; state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.DecimalValue: if (state.CurrentChar == 'e' || state.CurrentChar == 'E') { state.CurrentTokenizationType = SqlTokenizationType.FloatValue; state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.FloatValue: if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.ConsumeCurrentCharacterIntoToken(); } else if ((state.CurrentChar == '-' || state.CurrentChar == '+') && state.CurrentTokenValue.ToString().ToUpper().EndsWith("E")) { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.BinaryValue: if ((state.CurrentChar >= '0' && state.CurrentChar <= '9') || (state.CurrentChar >= 'A' && state.CurrentChar <= 'F') || (state.CurrentChar >= 'a' && state.CurrentChar <= 'f') ) { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleDollar: state.CurrentTokenValue.Append('$'); if ((state.CurrentChar >= 'A' && state.CurrentChar <= 'Z') || (state.CurrentChar >= 'a' && state.CurrentChar <= 'z') ) { state.CurrentTokenizationType = SqlTokenizationType.PseudoName; } else { state.CurrentTokenizationType = SqlTokenizationType.MonetaryValue; } state.ConsumeCurrentCharacterIntoToken(); break; case SqlTokenizationType.MonetaryValue: if (state.CurrentChar >= '0' && state.CurrentChar <= '9') { state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == '-' && state.CurrentTokenValue.Length == 1) { state.ConsumeCurrentCharacterIntoToken(); } else if (state.CurrentChar == '.' && !state.CurrentTokenValue.ToString().Contains(".")) { state.ConsumeCurrentCharacterIntoToken(); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleHyphen: if (state.CurrentChar == '-') { state.CurrentTokenizationType = SqlTokenizationType.SingleLineComment; state.HasUnprocessedCurrentCharacter = false; //DISCARDING the hyphen because of weird standard } else if (state.CurrentChar == '=') { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; state.CurrentTokenValue.Append('-'); AppendCharAndCompleteToken(state); } else { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; state.CurrentTokenValue.Append('-'); CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleSlash: if (state.CurrentChar == '*') { state.CurrentTokenizationType = SqlTokenizationType.BlockComment; state.HasUnprocessedCurrentCharacter = false; //DISCARDING the asterisk because of weird standard state.CommentNesting++; } else if (state.CurrentChar == '/') { state.CurrentTokenizationType = SqlTokenizationType.SingleLineCommentCStyle; state.HasUnprocessedCurrentCharacter = false; //DISCARDING the slash because of weird standard } else if (state.CurrentChar == '=') { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; state.CurrentTokenValue.Append('/'); AppendCharAndCompleteToken(state); } else { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; state.CurrentTokenValue.Append('/'); CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleLineComment: case SqlTokenizationType.SingleLineCommentCStyle: if (state.CurrentChar == (char)13 || state.CurrentChar == (char)10) { int nextCharInt = state.InputReader.Peek(); if (state.CurrentChar == (char)13 && nextCharInt == 10) { state.ConsumeCurrentCharacterIntoToken(); state.ReadNextCharacter(); } AppendCharAndCompleteToken(state); } else { state.ConsumeCurrentCharacterIntoToken(); } break; case SqlTokenizationType.BlockComment: if (state.CurrentChar == '*') { if (state.InputReader.Peek() == (int)'/') { state.CommentNesting--; if (state.CommentNesting > 0) { state.ConsumeCurrentCharacterIntoToken(); state.ReadNextCharacter(); state.ConsumeCurrentCharacterIntoToken(); } else { state.HasUnprocessedCurrentCharacter = false; //discarding the asterisk state.ReadNextCharacter(); //TODO: DANGER DANGER why do "contained" token types have this inconsistent handling where the delimiters are not in the value??? SwallowOutstandingCharacterAndCompleteToken(state); } } else { state.ConsumeCurrentCharacterIntoToken(); } } else { if (state.CurrentChar == '/' && state.InputReader.Peek() == (int)'*') { state.ConsumeCurrentCharacterIntoToken(); state.ReadNextCharacter(); state.ConsumeCurrentCharacterIntoToken(); state.CommentNesting++; } else { state.ConsumeCurrentCharacterIntoToken(); } } break; case SqlTokenizationType.OtherNode: case SqlTokenizationType.PseudoName: if (IsNonWordCharacter(state.CurrentChar)) { CompleteTokenAndProcessNext(state); } else { state.ConsumeCurrentCharacterIntoToken(); } break; case SqlTokenizationType.SingleN: if (state.CurrentChar == '\'') { state.CurrentTokenizationType = SqlTokenizationType.NString; state.HasUnprocessedCurrentCharacter = false; //DISCARDING the apostrophe because of weird standard } else { if (IsNonWordCharacter(state.CurrentChar)) { CompleteTokenAndProcessNext(state); } else { state.CurrentTokenizationType = SqlTokenizationType.OtherNode; state.CurrentTokenValue.Append('N'); state.ConsumeCurrentCharacterIntoToken(); } } break; case SqlTokenizationType.NString: case SqlTokenizationType.String: if (state.CurrentChar == '\'') { if (state.InputReader.Peek() == (int)'\'') { //add the character (once) state.ConsumeCurrentCharacterIntoToken(); //throw away the second character... because (for some reason?) we're storing the effective value" rather than the raw token... state.DiscardNextCharacter(); } else { //TODO: DANGER DANGER why do "contained" token types have this inconsistent handling where the delimiters are not in the value??? SwallowOutstandingCharacterAndCompleteToken(state); } } else { state.ConsumeCurrentCharacterIntoToken(); } break; case SqlTokenizationType.QuotedString: if (state.CurrentChar == '"') { if (state.InputReader.Peek() == (int)'"') { //add the character (once) state.ConsumeCurrentCharacterIntoToken(); //throw away the second character... because (for some reason?) we're storing the effective value" rather than the raw token... state.DiscardNextCharacter(); } else { //TODO: DANGER DANGER why do "contained" token types have this inconsistent handling where the delimiters are not in the value??? SwallowOutstandingCharacterAndCompleteToken(state); } } else { state.ConsumeCurrentCharacterIntoToken(); } break; case SqlTokenizationType.BracketQuotedName: if (state.CurrentChar == ']') { if (state.InputReader.Peek() == (int)']') { //add the character (once) state.ConsumeCurrentCharacterIntoToken(); //throw away the second character... because (for some reason?) we're storing the effective value" rather than the raw token... state.DiscardNextCharacter(); } else { //TODO: DANGER DANGER why do "contained" token types have this inconsistent handling where the delimiters are not in the value??? SwallowOutstandingCharacterAndCompleteToken(state); } } else { state.ConsumeCurrentCharacterIntoToken(); } break; case SqlTokenizationType.SingleLT: state.CurrentTokenValue.Append('<'); state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; if (state.CurrentChar == '=' || state.CurrentChar == '>' || state.CurrentChar == '<') { AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleGT: state.CurrentTokenValue.Append('>'); state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; if (state.CurrentChar == '=' || state.CurrentChar == '>') { AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleAsterisk: state.CurrentTokenValue.Append('*'); if (state.CurrentChar == '=') { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleOtherCompoundableOperator: state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; if (state.CurrentChar == '=') { AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SinglePipe: state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; state.CurrentTokenValue.Append('|'); if (state.CurrentChar == '=' || state.CurrentChar == '|') { AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleEquals: state.CurrentTokenValue.Append('='); if (state.CurrentChar == '=') { AppendCharAndCompleteToken(state); } else { CompleteTokenAndProcessNext(state); } break; case SqlTokenizationType.SingleExclamation: state.CurrentTokenValue.Append('!'); if (state.CurrentChar == '=' || state.CurrentChar == '<' || state.CurrentChar == '>') { state.CurrentTokenizationType = SqlTokenizationType.OtherOperator; AppendCharAndCompleteToken(state); } else { state.CurrentTokenizationType = SqlTokenizationType.OtherNode; CompleteTokenAndProcessNext(state); } break; default: throw new Exception("In-progress node unrecognized!"); } state.ReadNextCharacter(); } if (state.CurrentTokenizationType != null) { if (state.CurrentTokenizationType.Value == SqlTokenizationType.BlockComment || state.CurrentTokenizationType.Value == SqlTokenizationType.String || state.CurrentTokenizationType.Value == SqlTokenizationType.NString || state.CurrentTokenizationType.Value == SqlTokenizationType.QuotedString || state.CurrentTokenizationType.Value == SqlTokenizationType.BracketQuotedName ) { state.TokenContainer.HasUnfinishedToken = true; } SwallowOutstandingCharacterAndCompleteToken(state); } return(state.TokenContainer); }
protected override IEnumerable <Result <FilterExpressionToken> > Tokenize( TextSpan stringSpan, TokenizationState <FilterExpressionToken> tokenizationState) { var next = SkipWhiteSpace(stringSpan); if (!next.HasValue) { yield break; } do { if (char.IsDigit(next.Value)) { var hex = FilterExpressionTextParsers.HexInteger(next.Location); if (hex.HasValue) { next = hex.Remainder.ConsumeChar(); yield return(Result.Value(FilterExpressionToken.HexNumber, hex.Location, hex.Remainder)); } else { var real = FilterExpressionTextParsers.Real(next.Location); if (!real.HasValue) { yield return(Result.CastEmpty <TextSpan, FilterExpressionToken>(real)); } else { yield return(Result.Value(FilterExpressionToken.Number, real.Location, real.Remainder)); } next = real.Remainder.ConsumeChar(); } if (!IsDelimiter(next)) { yield return(Result.Empty <FilterExpressionToken>(next.Location, new[] { "digit" })); } } else if (next.Value == '\'') { var str = FilterExpressionTextParsers.SqlString(next.Location); if (!str.HasValue) { yield return(Result.CastEmpty <string, FilterExpressionToken>(str)); } next = str.Remainder.ConsumeChar(); yield return(Result.Value(FilterExpressionToken.String, str.Location, str.Remainder)); } else if (next.Value == '@') { var beginIdentifier = next.Location; var startOfName = next.Remainder; do { next = next.Remainder.ConsumeChar(); }while (next.HasValue && char.IsLetterOrDigit(next.Value)); if (next.Remainder == startOfName) { yield return(Result.Empty <FilterExpressionToken>(startOfName, new[] { "built-in identifier name" })); } else { yield return(Result.Value(FilterExpressionToken.BuiltInIdentifier, beginIdentifier, next.Location)); } } else if (char.IsLetter(next.Value) || next.Value == '_') { var beginIdentifier = next.Location; do { next = next.Remainder.ConsumeChar(); }while (next.HasValue && (char.IsLetterOrDigit(next.Value) || next.Value == '_')); FilterExpressionToken keyword; if (TryGetKeyword(beginIdentifier.Until(next.Location), out keyword)) { yield return(Result.Value(keyword, beginIdentifier, next.Location)); } else { yield return(Result.Value(FilterExpressionToken.Identifier, beginIdentifier, next.Location)); } } else if (next.Value == '/' && (!tokenizationState.Previous.HasValue || PreRegexTokens.Contains(tokenizationState.Previous.Value.Kind))) { var regex = FilterExpressionTextParsers.RegularExpression(next.Location); if (!regex.HasValue) { yield return(Result.CastEmpty <Unit, FilterExpressionToken>(regex)); } yield return(Result.Value(FilterExpressionToken.RegularExpression, next.Location, regex.Remainder)); next = regex.Remainder.ConsumeChar(); } else { var compoundOp = FilterExpressionTextParsers.CompoundOperator(next.Location); if (compoundOp.HasValue) { yield return(Result.Value(compoundOp.Value, compoundOp.Location, compoundOp.Remainder)); next = compoundOp.Remainder.ConsumeChar(); } else if (next.Value < SimpleOps.Length && SimpleOps[next.Value] != FilterExpressionToken.None) { yield return(Result.Value(SimpleOps[next.Value], next.Location, next.Remainder)); next = next.Remainder.ConsumeChar(); } else { yield return(Result.Empty <FilterExpressionToken>(next.Location)); next = next.Remainder.ConsumeChar(); } } next = SkipWhiteSpace(next.Location); } while (next.HasValue); }
protected override IEnumerable <Result <TokenType> > Tokenize(TextSpan remainder, TokenizationState <TokenType> state) { // Parsing of the properties file syntax has to be stateful: any string can appear inside the value, so // without knowing what the last token was, you don't know how to parse the remaining text. Hence why // we're overriding this overload that takes the state. while (!remainder.IsAtEnd) { var parserToTry = GetParserToTry(state); var parseResult = parserToTry(remainder); if (parseResult.HasValue) { remainder = parseResult.Remainder; yield return(parseResult); } else { yield return(Result.Empty <TokenType>(remainder)); } } }
protected override IEnumerable <Result <CspTokenType> > Tokenize(TextSpan remainder, TokenizationState <CspTokenType> state) { while (true) { var next = remainder.ConsumeChar(); if (!next.HasValue) { yield break; } if (next.Value == ';') { yield return(Result.Value( CspTokenType.Semicolon, next.Location, next.Remainder )); remainder = next.Remainder; } else if (next.Value == ' ') { var consumed = WhitespaceParser(remainder); yield return(Result.Value( CspTokenType.Whitespace, consumed.Location, consumed.Remainder )); remainder = consumed.Remainder; } else if (IsValidFirstCharForDirectiveNameOrSourceExpression(next.Value)) { var consumed = DirectiveNameOrSourceExpressionParser(remainder); yield return(Result.Value( CspTokenType.DirectiveNameOrSourceExpression, consumed.Location, consumed.Remainder )); remainder = consumed.Remainder; } else { // Couldn't parse. yield return(Result.Empty <CspTokenType>(remainder)); } } }
/// <summary> /// Subclasses should override to perform tokenization when the /// last-produced-token needs to be tracked. /// </summary> /// <param name="span">The input span to tokenize.</param> /// <param name="state">The tokenization state maintained during the operation.</param> /// <returns>A list of parsed tokens.</returns> protected virtual IEnumerable<Result<TKind>> Tokenize(TextSpan span, TokenizationState<TKind> state) { return Tokenize(span); }