private static void CreateTree_AndOr(TokenSet tokens)
{
//treat "and" and "or" as operators too
TokenEnumerator enumerator = tokens.GetEnumerator();
while (enumerator.MoveNext())
{
//make sure this is the start of a new group
Token starter = enumerator.Current;
if (starter.Type != TokenType.Keyword || starter.Children.Count > 1 ||
(starter.Value.ToLower() != "and" && starter.Value.ToLower() != "or"))
{
CreateTree_AndOr(starter.Children);
continue;
}
if (starter.Children.Count == 0 && enumerator.Next != null)
{
starter.Children.Add(enumerator.Next);
enumerator.RemoveNext();
}
if (enumerator.Previous != null)
{
starter.Children.AddToBeginning(enumerator.Previous);
enumerator.RemovePrevious();
}
CreateTree_AndOr(starter.Children);
}
}
private static void IdentifySpecialTokens(TokenSet tokens)
{
//identify the tokens
foreach (Token token in tokens)
{
//only work on unidentified tokens
if (token.Type != TokenType.Unknown)
{
continue;
}
//identify the operators
if (token.Value.Length == 1 && OPERATORS.IndexOf(token.Value) > -1)
{
token.Type = TokenType.Operator;
continue;
}
//identify variables
if (token.Value.StartsWith("@"))
{
token.Type = TokenType.Variable;
continue;
}
//pull other types
switch (token.Value.ToLower())
{
case ".":
token.Type = TokenType.Dot;
break;
case ";":
token.Type = TokenType.Semicolon;
break;
case "begin":
case "(":
token.Type = TokenType.GroupBegin;
break;
case "case":
token.Type = TokenType.CaseStatement;
break;
case "end":
case ")":
token.Type = TokenType.GroupEnd;
break;
case "add":
case "alter":
case "and":
case "clustered":
case "collate":
case "constraint":
case "create":
case "default":
case "drop":
case "else":
case "exists":
case "for":
case "from":
case "function":
case "go":
case "identity":
case "if":
case "in":
case "index":
case "is":
case "key":
case "nocheck":
case "nonclustered":
case "not":
case "null":
case "on":
case "or":
case "proc":
case "procedure":
case "primary":
case "select":
case "table":
case "tran":
case "transaction":
case "trigger":
case "then":
case "unique":
case "view":
case "when":
case "where":
case "with":
token.Type = TokenType.Keyword;
break;
case ",":
token.Type = TokenType.Separator;
break;
case "'":
case "\"":
token.Type = TokenType.Quote;
break;
}
}
//fix misidentified tokens
TokenEnumerator enumerator = tokens.GetEnumerator();
while (enumerator.MoveNext())
{
Token previous = enumerator.Previous;
Token current = enumerator.Current;
Token next = enumerator.Next;
if (current.Type == TokenType.GroupBegin && next.Type == TokenType.Keyword &&
current.Value.ToLower() == "begin" && next.Value.ToLower().StartsWith("tran"))
{
current.Type = TokenType.Keyword;
}
else if (previous != null && current.Type == TokenType.GroupBegin &&
previous.Type == TokenType.Quote && next.Type == TokenType.Quote)
{
current.Type = TokenType.StringValue;
}
}
//coalese operators and things with dots
enumerator = tokens.GetEnumerator();
enumerator.MoveLast();
while (enumerator.MovePrevious())
{
Token previous = enumerator.Previous;
Token current = enumerator.Current;
Token next = enumerator.Next;
//do the coalesce but don't screw up +5 * -2
if (next != null && current.Type == TokenType.Operator && next.Type == TokenType.Operator &&
next.Value != "-" && next.Value != "+")
{
current.Value = current.Value + next.Value;
enumerator.RemoveNext();
continue;
}
else if (current.Type == TokenType.Dot && previous != null && next != null &&
(previous.Type == TokenType.Unknown || previous.Type == TokenType.Identifier || previous.Type == TokenType.Dot) &&
(next.Type == TokenType.Unknown || next.Type == TokenType.Identifier || next.Type == TokenType.Dot))
{
current.StartIndex = previous.StartIndex;
current.Value = previous.FlattenTree() + "." + next.FlattenTree();
enumerator.RemovePrevious();
enumerator.RemoveNext();
continue;
}
}
}
private static void CreateTree_Operator(TokenSet tokens)
{
//work on operators
TokenEnumerator enumerator = tokens.GetEnumerator();
while (enumerator.MoveNext())
{
//make sure this is the start of a new group
Token starter = enumerator.Current;
if (starter.Type != TokenType.Operator || starter.Children.Count > 0)
{
CreateTree_Operator(starter.Children);
continue;
}
Token previous = null;
if (enumerator.Previous != null && starter.Value != "!" && starter.Value != "~")
{
previous = enumerator.Previous;
}
Token next = enumerator.Next;
//don't bury keywords in the tree
if (previous != null && (previous.Type == TokenType.Keyword || previous.Type == TokenType.Separator))
{
CreateTree_Operator(starter.Children);
continue;
}
//add previous operand if not unary (don't remove - screws up adding of next operand)
if (previous != null && previous.Type != TokenType.Comment && previous.Type != TokenType.Keyword &&
previous.Type != TokenType.StringValue)
{
starter.Children.Add(previous);
}
else
{
previous = null;
}
//add next operand
while (next != null)
{
starter.Children.Add(next);
enumerator.RemoveNext();
if (next.Type == TokenType.Operator)
{
next = enumerator.Next;
}
else
{
next = null;
}
}
//remove any previous operand
if (previous != null)
{
enumerator.RemovePrevious();
}
//make a tree of those children too
CreateTree_Operator(starter.Children);
}
}
