private void ConsumeUnquotedToken()
{
int startPos = pos;
while (pos < fullSql.Length && !IsSpecialCharacter(fullSql[pos]))
pos++;
string word = fullSql.Substring(startPos, pos - startPos);
double v;
if (Double.TryParse(word, out v))
tokens.Add(new Token(TokenType.Number, "?"));
else
{
Token t = new Token(TokenType.Identifier, word);
if (IsKeyword(word))
{
t.Type = TokenType.Keyword;
t.Text = t.Text.ToUpperInvariant();
}
tokens.Add(t);
}
}
private void CollapseValueList(List<Token> tok, ref int pos)
{
List<int> parenIndices = new List<int>();
// this while loop will find all closing parens in this value list
while (true)
{
// find the close ')'
while (++pos < tok.Count)
if (tok[pos].Type == TokenType.Symbol && tok[pos].Text == ")") break;
Debug.Assert(pos < tok.Count);
parenIndices.Add(pos);
// now find the next "real" token
while (++pos < tok.Count)
if (tok[pos].IsRealToken) break;
if (pos == tok.Count) break;
if (tok[pos].Text != ",")
{
pos--;
break;
}
}
// if we only have 1 value then we don't collapse
if (parenIndices.Count < 2) return;
int index = parenIndices[0];
tok[++index] = new Token(TokenType.Whitespace, " ");
tok[++index] = new Token(TokenType.Comment, "/* , ... */");
index++;
// now mark all the other tokens as no output
while (index <= parenIndices[parenIndices.Count - 1])
tok[index++].Output = false;
}