private IAToken GetQuotedIdentifier(ref int textPos, char endQuote)
{
int startPos = textPos;
textPos++;
while (textPos < _text.Length)
{
if (_text[textPos] == endQuote)
{
break;
}
textPos++;
}
if (textPos == _text.Length)
{
throw new ATokenizerException(string.Format("Missing terminating quote for quoted identifier starting at position {0}", startPos));
}
textPos++;
IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, textPos - startPos));
SetAddSpaceFlag(token, textPos);
return(token);
}
private IAToken CreateToken(int startPos, int endPos)
{
IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, endPos - startPos));
SetAddSpaceFlag(token, endPos);
return(token);
}
public void Tokenize(string text)
{
_text = text;
_parenthesesCounter = 0;
_curlyBracesCounter = 0;
int textPos = 0;
IAToken token = GetNextToken(ref textPos);
while (token != null)
{
Tokens.Add(token);
token = GetNextToken(ref textPos);
}
CheckThatParenthesesMatch();
}
private IAToken GetStringToken(ref int textPos)
{
int startPos = textPos;
textPos++;
while (textPos < _text.Length)
{
if (_text[textPos] == QuoteChar)
{
if (textPos + 1 < _text.Length && _text[textPos + 1] == QuoteChar)
{
textPos += 2;
}
else
{
break;
}
}
else
{
textPos++;
}
}
if (textPos == _text.Length)
{
throw new ATokenizerException(string.Format("Missing terminating quote for string starting at position {0}", startPos));
}
textPos++;
IAToken token = TokenFactory.CreateToken(_text.Substring(startPos, textPos - startPos));
if (ExpandEmptyStrings && token.Text == "''")
{
token.Text = "' '";
}
SetAddSpaceFlag(token, textPos);
return(token);
}
private void SetAddSpaceFlag(IAToken token, int textPos)
{
token.AddSpace = textPos < _text.Length && IsWhitespace(_text[textPos]);
}
