private int ProcessToken(int index, List <Token> tokens, ConditionBuilder <T> builder)
{
Token token = tokens[index];
if (TokenConstants.IsJoinOperator(token))
{
if (token.Value == TokenConstants.CONST_AND)
{
builder.And();
}
else if (token.Value == TokenConstants.CONST_OR)
{
builder.Or();
}
}
return(0);
}
public List <Token> Tokenize(string query)
{
Contract.Requires(!String.IsNullOrWhiteSpace(query));
List <Token> tokens = new List <Token>();
bool quoted = false;
StringBuilder sb = null;
for (int index = 0; index < query.Length; index++)
{
char cc = query[index];
if (Char.IsWhiteSpace(cc))
{
if (quoted)
{
if (sb == null)
{
throw new TokenizerException(String.Format("Invalid Tokenizer State: expected value buffer. [string={0}][index={1}]", query, index));
}
sb.Append(cc);
}
}
else
{
Token tk = TokenConstants.ParseToken(query, index);
if (!quoted && tk != null)
{
if (sb != null)
{
ValueToken t = ValueToken(sb, index);
tokens.Add(t);
sb = null;
}
tokens.Add(tk);
index = tk.Index + tk.Length - 1;
continue;
}
else
{
if (cc == TokenConstants.CONST_DOUBLE_QUOTE || cc == TokenConstants.CONST_SINGLE_QUOTE)
{
if (quoted)
{
char pc = query[index - 1];
if (pc == '\\')
{
if (sb == null)
{
throw new TokenizerException(String.Format("Invalid Tokenizer State: expected value buffer. [string={0}][index={1}]", query, index));
}
sb.Append(cc);
}
else
{
if (sb == null)
{
throw new TokenizerException(String.Format("Invalid Tokenizer State: expected value buffer. [string={0}][index={1}]", query, index));
}
quoted = false;
StringToken vt = new StringToken();
vt.Value = sb.ToString();
vt.Index = index - vt.Value.Length;
vt.Length = vt.Value.Length;
tokens.Add(vt);
sb = null;
}
}
else
{
quoted = true;
if (sb != null)
{
ValueToken t = ValueToken(sb, index);
tokens.Add(t);
sb = null;
}
sb = new StringBuilder();
}
}
else if (cc == TokenConstants.CONST_COMMA[0])
{
if (sb != null)
{
ValueToken vt = ValueToken(sb, index);
tokens.Add(vt);
sb = null;
}
ListSeperatorToken t = new ListSeperatorToken();
t.Index = index;
t.Length = t.Value.Length;
tokens.Add(t);
}
else
{
if (sb != null)
{
sb.Append(cc);
}
else
{
sb = new StringBuilder();
sb.Append(cc);
}
}
}
}
}
if (tokens.Count > 0)
{
return(tokens);
}
return(null);
}
