private IAToken GetQuotedIdentifier(ref int textPos, char endQuote) { int startPos = textPos; textPos++; while (textPos < _text.Length) { if (_text[textPos] == endQuote) { break; } textPos++; } if (textPos == _text.Length) { throw new ATokenizerException(string.Format("Missing terminating quote for quoted identifier starting at position {0}", startPos)); } textPos++; IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, textPos - startPos)); SetAddSpaceFlag(token, textPos); return(token); }
private IAToken CreateToken(int startPos, int endPos) { IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, endPos - startPos)); SetAddSpaceFlag(token, endPos); return(token); }
public void Tokenize(string text) { _text = text; _parenthesesCounter = 0; _curlyBracesCounter = 0; int textPos = 0; IAToken token = GetNextToken(ref textPos); while (token != null) { Tokens.Add(token); token = GetNextToken(ref textPos); } CheckThatParenthesesMatch(); }
private IAToken GetStringToken(ref int textPos) { int startPos = textPos; textPos++; while (textPos < _text.Length) { if (_text[textPos] == QuoteChar) { if (textPos + 1 < _text.Length && _text[textPos + 1] == QuoteChar) { textPos += 2; } else { break; } } else { textPos++; } } if (textPos == _text.Length) { throw new ATokenizerException(string.Format("Missing terminating quote for string starting at position {0}", startPos)); } textPos++; IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, textPos - startPos)); if (ExpandEmptyStrings && token.Text == "''") { token.Text = "' '"; } SetAddSpaceFlag(token, textPos); return(token); }
private void SetAddSpaceFlag(IAToken token, int textPos) { token.AddSpace = textPos < _text.Length && IsWhitespace(_text[textPos]); }