Esempio n. 1
0
        private static List <int> ParseReleaseCategory(ITokenList classes)
        {
            var result = new List <int>();

            if (classes.Contains("fa-gamepad"))
            {
                result.Add(TorznabCatType.PCGames.ID);
            }
            else if (classes.Contains("fa-film"))
            {
                result.Add(TorznabCatType.Movies.ID);
            }
            else if (classes.Contains("fa-tv"))
            {
                result.Add(TorznabCatType.TV.ID);
            }
            else if (classes.Contains("fa-microchip"))
            {
                result.Add(TorznabCatType.PC.ID);
            }
            else if (classes.Contains("fa-android"))
            {
                result.Add(TorznabCatType.PCPhoneAndroid.ID);
            }
            return(result);
        }
Esempio n. 2
0
        private bool ClassListsAreEqual(ITokenList classListX, ITokenList classListY)
        {
            if (_options.IgnoreAdditionalClassNames)
            {
                if (classListX.Length > classListY.Length)
                {
                    return(false);
                }

                if (_options.IgnoreClassNameOrder)
                {
                    return(classListX.Intersect(classListY, StringComparer.Ordinal).Count() == classListX.Length);
                }

                return(classListX.SequenceEqual(classListY.Intersect(classListX), StringComparer.Ordinal));
            }

            // Check exact element match if no additional class names are allowed
            if (classListX.Length != classListY.Length)
            {
                return(false);
            }

            if (_options.IgnoreClassNameOrder)
            {
                return(classListX.Union(classListY, StringComparer.Ordinal).Count() == classListX.Length);
            }

            return(classListX.SequenceEqual(classListY, StringComparer.Ordinal));
        }
Esempio n. 3
0
        public void StandardFormatReparsingReformatting(string FileName)
        {
            string inputSQL = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            TSqlStandardFormatter _treeFormatter = GetFormatter("");
            ITokenList            tokenized      = _tokenizer.TokenizeSQL(inputSQL);
            Node   parsed    = _parser.ParseSQL(tokenized);
            string outputSQL = _treeFormatter.FormatSQLTree(parsed);

            var inputToSecondPass = outputSQL;

            if (inputToSecondPass.StartsWith(Utils.ERROR_FOUND_WARNING))
            {
                inputToSecondPass = inputToSecondPass.Replace(Utils.ERROR_FOUND_WARNING, "");
            }

            ITokenList tokenizedAgain = _tokenizer.TokenizeSQL(inputToSecondPass);
            Node       parsedAgain    = _parser.ParseSQL(tokenizedAgain);
            string     formattedAgain = _treeFormatter.FormatSQLTree(parsedAgain);

            if (!inputSQL.Contains(Utils.REFORMATTING_INCONSISTENCY_WARNING))
            {
                Assert.AreEqual(outputSQL, formattedAgain, "first-pass formatted vs reformatted");
                Utils.StripWhiteSpaceFromSqlTree(parsed);
                Utils.StripWhiteSpaceFromSqlTree(parsedAgain);
                Assert.AreEqual(parsed.ToXmlDoc().OuterXml.ToUpper(), parsedAgain.ToXmlDoc().OuterXml.ToUpper(), "first parse xml vs reparse xml");
            }
        }
Esempio n. 4
0
        private ITokenList Validate(ITokenList tokens)
        {
            int listDepth = 0;

            foreach (var token in tokens)
            {
                if (token.Type == TokenType.Parenthesis)
                {
                    if (token.Value == "(")
                    {
                        listDepth++;
                    }
                    else if (token.Value == ")")
                    {
                        listDepth--;
                    }
                }
            }

            if (listDepth != 0)
            {
                throw new Exception("TODO");
            }

            return(tokens);
        }
        public void ContentUnchangedByIdentityTokenFormatter(string FileName)
        {
            string     inputSQL  = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            ITokenList tokenized = _tokenizer.TokenizeSQL(inputSQL);
            string     outputSQL = _tokenFormatter.FormatSQLTokens(tokenized);

            Assert.AreEqual(inputSQL, outputSQL);
        }
        private void TestMarkerPosition(string inputSQLNoLineBreaks, int inputPosition)
        {
            ITokenList tokenized = _tokenizer.TokenizeSQL(inputSQLNoLineBreaks, inputPosition);

            Assert.AreEqual(SqlTokenType.OtherNode, tokenized.MarkerToken.Type, "token type");
            Assert.AreEqual("from", tokenized.MarkerToken.Value, "token value");
            Assert.AreEqual(2, tokenized.MarkerPosition);
        }
Esempio n. 7
0
        public void ContentUnchangedByIdentityTokenFormatter(string FileName)
        {
            string     inputSQL  = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            ITokenList tokenized = _tokenizer.TokenizeSQL(inputSQL);
            string     outputSQL = _tokenFormatter.FormatSQLTokens(tokenized);

            if (!inputSQL.Contains(Utils.INVALID_SQL_WARNING))
            {
                Assert.AreEqual(outputSQL, inputSQL);
            }
        }
Esempio n. 8
0
 public static bool Contains(this ITokenList list, string[] tokens)
 {
     for (int i = 0; i < tokens.Length; i++)
     {
         if (!list.Contains(tokens[i]))
         {
             return(false);
         }
     }
     return(true);
 }
        public void StandardFormatExpectedOutput(string FileName)
        {
            string expectedSql = Utils.GetTestFileContent(FileName, Utils.STANDARDFORMATSQLFOLDER);
            string inputSql    = Utils.GetTestFileContent(Utils.StripFileConfigString(FileName), Utils.INPUTSQLFOLDER);
            TSqlStandardFormatter _treeFormatter = GetFormatter(Utils.GetFileConfigString(FileName));

            ITokenList  tokenized = _tokenizer.TokenizeSQL(inputSql);
            XmlDocument parsed    = _parser.ParseSQL(tokenized);
            string      formatted = _treeFormatter.FormatSQLTree(parsed);

            Assert.AreEqual(expectedSql, formatted);
        }
 public void CheckThatValidSqlRemainsUnchangedByIdentityTokenFormatter()
 {
     foreach (string inputSQL in Utils.FolderTextFileIterator(TestDataFolder))
     {
         ITokenList tokenized = _tokenizer.TokenizeSQL(inputSQL);
         string     outputSQL = _tokenFormatter.FormatSQLTokens(tokenized);
         if (!inputSQL.Contains("THIS TEST FILE IS NOT VALID SQL"))
         {
             Assert.AreEqual <string>(outputSQL, inputSQL, "input and output should be the same, as this is a valid SQL file");
         }
     }
 }
Esempio n. 11
0
        /// <summary>
        /// Returns true if the underlying string contains all of the tokens, otherwise false.
        /// </summary>
        /// <param name="list">The list that is considered.</param>
        /// <param name="tokens">The tokens to consider.</param>
        /// <returns>True if the string contained all tokens, otherwise false.</returns>
        public static Boolean Contains(this ITokenList list, String[] tokens)
        {
            for (var i = 0; i < tokens.Length; i++)
            {
                if (!list.Contains(tokens[i]))
                {
                    return(false);
                }
            }

            return(true);
        }
        public void ExpectedParseTree(string FileName)
        {
            XmlDocument expectedXmlDoc = new XmlDocument();

            expectedXmlDoc.PreserveWhitespace = true;
            expectedXmlDoc.Load(Path.Combine(Utils.GetTestContentFolder(Utils.PARSEDSQLFOLDER), FileName));
            string inputSql = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);

            ITokenList tokenized = _tokenizer.TokenizeSQL(inputSql);
            Node       parsed    = _parser.ParseSQL(tokenized);

            Assert.AreEqual(expectedXmlDoc.OuterXml, parsed.ToXmlDoc().OuterXml);
        }
        public void CheckThatStandardOutputSqlMatchesExpectedStandardOutputSql()
        {
            foreach (FileInfo expectedFormatFile in new DirectoryInfo(FormattedDataFolder).GetFiles())
            {
                string expectedSql = File.ReadAllText(expectedFormatFile.FullName);
                string inputSql    = File.ReadAllText(Path.Combine(InputDataFolder, expectedFormatFile.Name));

                ITokenList  tokenized = _tokenizer.TokenizeSQL(inputSql);
                XmlDocument parsed    = _parser.ParseSQL(tokenized);
                string      formatted = _treeFormatter.FormatSQLTree(parsed);

                Assert.AreEqual <string>(expectedSql, formatted, string.Format("Formatted Sql does not match expected result for file {0}", expectedFormatFile.Name));
            }
        }
Esempio n. 14
0
        public void CheckThatParseTreeMatchesExpectedParseTree()
        {
            foreach (FileInfo xmlFile in new DirectoryInfo(ParsedDataFolder).GetFiles())
            {
                XmlDocument expectedXmlDoc = new XmlDocument();
                expectedXmlDoc.PreserveWhitespace = true;
                expectedXmlDoc.Load(xmlFile.FullName);
                string inputSql = File.ReadAllText(Path.Combine(InputDataFolder, xmlFile.Name));

                ITokenList  tokenized = _tokenizer.TokenizeSQL(inputSql);
                XmlDocument parsed    = _parser.ParseSQL(tokenized);

                Assert.AreEqual <string>(expectedXmlDoc.OuterXml, parsed.OuterXml, string.Format("Parse Tree Xml does not match for file {0}", xmlFile.Name));
            }
        }
 public void CheckThatReformattingOutputSqlYieldsSameSql()
 {
     foreach (string inputSQL in Utils.FolderTextFileIterator(InputDataFolder))
     {
         ITokenList  tokenized      = _tokenizer.TokenizeSQL(inputSQL);
         XmlDocument parsed         = _parser.ParseSQL(tokenized);
         string      outputSQL      = _treeFormatter.FormatSQLTree(parsed);
         ITokenList  tokenizedAgain = _tokenizer.TokenizeSQL(outputSQL);
         XmlDocument parsedAgain    = _parser.ParseSQL(tokenizedAgain);
         string      formattedAgain = _treeFormatter.FormatSQLTree(parsedAgain);
         if (!inputSQL.Contains("KNOWN SQL REFORMATTING INCONSISTENCY") && !inputSQL.Contains("THIS TEST FILE IS NOT VALID SQL"))
         {
             Assert.AreEqual <string>(outputSQL, formattedAgain, "reformatted SQL should be the same as first pass of formatting");
         }
     }
 }
 public void CheckThatReparsingOutputSqlYieldsEquivalentTree()
 {
     foreach (string inputSQL in Utils.FolderTextFileIterator(InputDataFolder))
     {
         ITokenList  tokenized      = _tokenizer.TokenizeSQL(inputSQL);
         XmlDocument parsed         = _parser.ParseSQL(tokenized);
         string      outputSQL      = _treeFormatter.FormatSQLTree(parsed);
         ITokenList  tokenizedAgain = _tokenizer.TokenizeSQL(outputSQL);
         XmlDocument parsedAgain    = _parser.ParseSQL(tokenizedAgain);
         Utils.StripWhiteSpaceFromSqlTree(parsed);
         Utils.StripWhiteSpaceFromSqlTree(parsedAgain);
         if (!inputSQL.Contains("KNOWN SQL REFORMATTING INCONSISTENCY") && !inputSQL.Contains("THIS TEST FILE IS NOT VALID SQL"))
         {
             Assert.AreEqual <string>(parsed.OuterXml.ToUpper(), parsedAgain.OuterXml.ToUpper(), "parsed SQL trees should be the same");
         }
     }
 }
        public void StandardFormatExpectedOutput(string fileName)
        {
            //string expectedSql = Utils.GetTestFileContent(fileName, Utils.STANDARDFORMATSQLFOLDER);


            string inputSql = Utils.GetTestFileContent(Utils.StripFileConfigString(fileName), Utils.INPUTSQLFOLDER);
            TSqlStandardFormatter treeFormatter = GetFormatter(Utils.GetFileConfigString(fileName));

            treeFormatter.Debug = false;
            ITokenList tokenized = _tokenizer.TokenizeSQL(inputSql);
            Node       parsed    = _parser.ParseSQL(tokenized);
            string     formatted = treeFormatter.FormatSQLTree(parsed);

            // System.Diagnostics.Debug.WriteLine(formatted);
            Utils.WriteTestFileContent($"formatted-{fileName}", Utils.STANDARDFORMATSQLFOLDER, formatted);
            //Assert.AreEqual(expectedSql, formatted);
        }
Esempio n. 18
0
        /// <summary>
        /// Performs a partial match on a list of tokens (e.g. classes on an element)
        /// </summary>
        /// <param name="tokenList">List of tokens to search in</param>
        /// <param name="filter">Partial token to match via an StartsWidth</param>
        /// <returns>First matching token if found, null if no match</returns>
        public static string PartialMatch(this ITokenList tokenList, string filter)
        {
            // No tokens then bail out
            if (tokenList.Length == 0)
            {
                return(null);
            }

            foreach (var token in tokenList)
            {
                if (token.StartsWith(filter, StringComparison.InvariantCultureIgnoreCase))
                {
                    return(token);
                }
            }

            return(null);
        }
Esempio n. 19
0
        public INode ParseExpression(ITokenList tokens)
        {
            if (tokens == null)
            {
                throw new ArgumentNullException("tokens");
            }

            var enumerator = tokens.GetEnumerator();

            while (enumerator.MoveNext())
            {
                if (enumerator.Current.Type != TokenType.Comment)
                {
                    return(ParseToken(enumerator));
                }
            }

            throw new ParsingException("Expression expected");
        }
        public void StandardFormatReparsingReformatting(string FileName)
        {
            string inputSQL = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            TSqlStandardFormatter _treeFormatter = GetFormatter("");
            ITokenList            tokenized      = _tokenizer.TokenizeSQL(inputSQL);
            XmlDocument           parsed         = _parser.ParseSQL(tokenized);
            string      outputSQL      = _treeFormatter.FormatSQLTree(parsed);
            ITokenList  tokenizedAgain = _tokenizer.TokenizeSQL(outputSQL);
            XmlDocument parsedAgain    = _parser.ParseSQL(tokenizedAgain);
            string      formattedAgain = _treeFormatter.FormatSQLTree(parsedAgain);

            if (!inputSQL.Contains(Utils.REFORMATTING_INCONSISTENCY_WARNING) && !inputSQL.Contains(Utils.INVALID_SQL_WARNING))
            {
                Assert.AreEqual(outputSQL, formattedAgain, "first-pass formatted vs reformatted");
                Utils.StripWhiteSpaceFromSqlTree(parsed);
                Utils.StripWhiteSpaceFromSqlTree(parsedAgain);
                Assert.AreEqual(parsed.OuterXml.ToUpper(), parsedAgain.OuterXml.ToUpper(), "first parse xml vs reparse xml");
            }
        }
        public void ObfuscatingFormatReformatMatch(string FileName)
        {
            string      inputSQL       = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            ITokenList  tokenized      = _tokenizer.TokenizeSQL(inputSQL);
            XmlDocument parsedOriginal = _parser.ParseSQL(tokenized);

            string      obfuscatedSql   = _obfuscatingFormatter.FormatSQLTree(parsedOriginal);
            ITokenList  tokenizedAgain  = _tokenizer.TokenizeSQL(obfuscatedSql);
            XmlDocument parsedAgain     = _parser.ParseSQL(tokenizedAgain);
            string      unObfuscatedSql = _standardFormatter.FormatSQLTree(parsedAgain);

            Utils.StripCommentsFromSqlTree(parsedOriginal);
            string standardFormattedSql = _standardFormatter.FormatSQLTree(parsedOriginal);

            if (!inputSQL.Contains(Utils.INVALID_SQL_WARNING))
            {
                Assert.AreEqual(standardFormattedSql, unObfuscatedSql, "standard-formatted vs obfuscatd and reformatted");
            }
        }
Esempio n. 22
0
        public IEnumerable <INode> ParseProgram(ITokenList tokens)
        {
            if (tokens == null)
            {
                throw new ArgumentNullException("tokens");
            }

            var nodes      = new List <INode>();
            var enumerator = tokens.GetEnumerator();

            while (enumerator.MoveNext())
            {
                if (enumerator.Current.Type != TokenType.Comment)
                {
                    nodes.Add(ParseToken(enumerator));
                }
            }

            return(nodes);
        }
Esempio n. 23
0
        private static void ParseAnsiClass(ITokenList tokens, AnsiTextAttribute attributes)
        {
            foreach (var token in tokens)
            {
                if (token == "tab")
                {
                    attributes.IsTab = true;
                    return;
                }

                if (!token.StartsWith("ansi-"))
                {
                    continue;
                }

                if (token.StartsWith("ansi-bg"))
                {
                    SetAnsiBackground(token, attributes);
                    continue;
                }
                if (token.StartsWith("ansi-fg"))
                {
                    SetAnsiForeground(token, attributes);
                    continue;
                }
                if (token.StartsWith("ansi-text"))
                {
                    SetAnsiAttribute(token, attributes);
                    continue;
                }

                switch (token)
                {
                case "ansi-cls":
                    attributes.IsCls = true;
                    break;
                }
            }
        }
Esempio n. 24
0
        public void ObfuscatingFormatReformatMatch(string FileName)
        {
            string     inputSQL       = Utils.GetTestFileContent(FileName, Utils.INPUTSQLFOLDER);
            ITokenList tokenized      = _tokenizer.TokenizeSQL(inputSQL);
            Node       parsedOriginal = _parser.ParseSQL(tokenized);
            string     obfuscatedSql  = _obfuscatingFormatter.FormatSQLTree(parsedOriginal);

            var inputToSecondPass = obfuscatedSql;

            if (inputToSecondPass.StartsWith(Utils.ERROR_FOUND_WARNING))
            {
                inputToSecondPass = inputToSecondPass.Replace(Utils.ERROR_FOUND_WARNING, "");
            }

            ITokenList tokenizedAgain  = _tokenizer.TokenizeSQL(inputToSecondPass);
            Node       parsedAgain     = _parser.ParseSQL(tokenizedAgain);
            string     unObfuscatedSql = _standardFormatter.FormatSQLTree(parsedAgain);

            Utils.StripCommentsFromSqlTree(parsedOriginal);
            string standardFormattedSql = _standardFormatter.FormatSQLTree(parsedOriginal);

            Assert.AreEqual(standardFormattedSql, unObfuscatedSql, "standard-formatted vs obfuscatd and reformatted");
        }
        private static void CompleteToken(ref SqlTokenizationType? currentTokenizationType, ITokenList tokenContainer, StringBuilder currentValue)
        {
            if (currentTokenizationType == null)
                throw new Exception("Cannot complete Token, as there is no current Tokenization Type");

            switch (currentTokenizationType)
            {


				case SqlTokenizationType.BlockComment:
                    tokenContainer.Add(new Token(SqlTokenType.MultiLineComment, currentValue.ToString()));
                    break;

                case SqlTokenizationType.OtherNode:
                    tokenContainer.Add(new Token(SqlTokenType.OtherNode, currentValue.ToString()));
                    break;

                case SqlTokenizationType.PseudoName:
                    tokenContainer.Add(new Token(SqlTokenType.PseudoName, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleLineComment:
                    tokenContainer.Add(new Token(SqlTokenType.SingleLineComment, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleLineCommentCStyle:
                    tokenContainer.Add(new Token(SqlTokenType.SingleLineCommentCStyle, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleHyphen:
                    tokenContainer.Add(new Token(SqlTokenType.OtherOperator, "-"));
                    break;

                case SqlTokenizationType.SingleDollar:
                    tokenContainer.Add(new Token(SqlTokenType.MonetaryValue, "$"));
                    break;

                case SqlTokenizationType.SingleSlash:
                    tokenContainer.Add(new Token(SqlTokenType.OtherOperator, "/"));
                    break;

                case SqlTokenizationType.WhiteSpace:
                    tokenContainer.Add(new Token(SqlTokenType.WhiteSpace, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleN:
                    tokenContainer.Add(new Token(SqlTokenType.OtherNode, "N"));
                    break;

                case SqlTokenizationType.SingleExclamation:
                    tokenContainer.Add(new Token(SqlTokenType.OtherNode, "!"));
                    break;

				case SqlTokenizationType.SinglePipe:
					tokenContainer.Add(new Token(SqlTokenType.OtherNode, "|"));
					break;

				case SqlTokenizationType.SingleGT:
					tokenContainer.Add(new Token(SqlTokenType.OtherOperator, ">"));
					break;

				case SqlTokenizationType.SingleLT:
					tokenContainer.Add(new Token(SqlTokenType.OtherOperator, "<"));
					break;

				case SqlTokenizationType.NString:
                    tokenContainer.Add(new Token(SqlTokenType.NationalString, currentValue.ToString()));
                    break;

                case SqlTokenizationType.String:
                    tokenContainer.Add(new Token(SqlTokenType.String, currentValue.ToString()));
                    break;

                case SqlTokenizationType.QuotedString:
                    tokenContainer.Add(new Token(SqlTokenType.QuotedString, currentValue.ToString()));
                    break;

                case SqlTokenizationType.BracketQuotedName:
                    tokenContainer.Add(new Token(SqlTokenType.BracketQuotedName, currentValue.ToString()));
                    break;

                case SqlTokenizationType.OtherOperator:
                case SqlTokenizationType.SingleOtherCompoundableOperator:
                    tokenContainer.Add(new Token(SqlTokenType.OtherOperator, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleZero:
                    tokenContainer.Add(new Token(SqlTokenType.Number, "0"));
                    break;

                case SqlTokenizationType.SinglePeriod:
                    tokenContainer.Add(new Token(SqlTokenType.Period, "."));
                    break;

                case SqlTokenizationType.SingleAsterisk:
                    tokenContainer.Add(new Token(SqlTokenType.Asterisk, currentValue.ToString()));
                    break;

                case SqlTokenizationType.SingleEquals:
                    tokenContainer.Add(new Token(SqlTokenType.EqualsSign, currentValue.ToString()));
                    break;

                case SqlTokenizationType.Number:
                case SqlTokenizationType.DecimalValue:
                case SqlTokenizationType.FloatValue:
                    tokenContainer.Add(new Token(SqlTokenType.Number, currentValue.ToString()));
                    break;

                case SqlTokenizationType.BinaryValue:
                    tokenContainer.Add(new Token(SqlTokenType.BinaryValue, currentValue.ToString()));
                    break;

                case SqlTokenizationType.MonetaryValue:
                    tokenContainer.Add(new Token(SqlTokenType.MonetaryValue, currentValue.ToString()));
                    break;


                default:
                    throw new Exception("Unrecognized SQL Node Type");
            }

            currentTokenizationType = null;
        }
        private static void ProcessOrOpenToken(ref SqlTokenizationType? currentTokenizationType, StringBuilder currentNodeValue, char currentCharacter, ITokenList tokenContainer)
        {

            if (currentTokenizationType != null)
                throw new Exception("Cannot start a new Token: existing Tokenization Type is not null");

            //start a new value.
            currentNodeValue.Length = 0;

            if (IsWhitespace(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.WhiteSpace;
                currentNodeValue.Append(currentCharacter);
            }
            else if (currentCharacter == '-')
            {
                currentTokenizationType = SqlTokenizationType.SingleHyphen;
            }
            else if (currentCharacter == '$')
            {
                currentTokenizationType = SqlTokenizationType.SingleDollar;
            }
            else if (currentCharacter == '/')
            {
                currentTokenizationType = SqlTokenizationType.SingleSlash;
            }
            else if (currentCharacter == 'N')
            {
                currentTokenizationType = SqlTokenizationType.SingleN;
            }
            else if (currentCharacter == '\'')
            {
                currentTokenizationType = SqlTokenizationType.String;
            }
            else if (currentCharacter == '"')
            {
                currentTokenizationType = SqlTokenizationType.QuotedString;
            }
            else if (currentCharacter == '[')
            {
                currentTokenizationType = SqlTokenizationType.BracketQuotedName;
            }
            else if (currentCharacter == '(')
            {
                tokenContainer.Add(new Token(SqlTokenType.OpenParens, currentCharacter.ToString()));
            }
            else if (currentCharacter == ')')
            {
                tokenContainer.Add(new Token(SqlTokenType.CloseParens, currentCharacter.ToString()));
            }
            else if (currentCharacter == ',')
            {
                tokenContainer.Add(new Token(SqlTokenType.Comma, currentCharacter.ToString()));
            }
            else if (currentCharacter == '.')
            {
                currentTokenizationType = SqlTokenizationType.SinglePeriod;
            }
            else if (currentCharacter == '0')
            {
                currentTokenizationType = SqlTokenizationType.SingleZero;
            }
            else if (currentCharacter >= '1' && currentCharacter <= '9')
            {
                currentTokenizationType = SqlTokenizationType.Number;
                currentNodeValue.Append(currentCharacter);
            }
            else if (IsCurrencyPrefix(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.MonetaryValue;
                currentNodeValue.Append(currentCharacter);
            }
            else if (currentCharacter == ';')
            {
                tokenContainer.Add(new Token(SqlTokenType.Semicolon, currentCharacter.ToString()));
            }
            else if (currentCharacter == ':')
            {
                tokenContainer.Add(new Token(SqlTokenType.Colon, currentCharacter.ToString()));
            }
            else if (currentCharacter == '*')
            {
				currentTokenizationType = SqlTokenizationType.SingleAsterisk;
			}
            else if (currentCharacter == '=')
            {
                currentTokenizationType = SqlTokenizationType.SingleEquals;
            }
            else if (currentCharacter == '<')
            {
                currentTokenizationType = SqlTokenizationType.SingleLT;
            }
            else if (currentCharacter == '>')
            {
                currentTokenizationType = SqlTokenizationType.SingleGT;
            }
            else if (currentCharacter == '!')
            {
                currentTokenizationType = SqlTokenizationType.SingleExclamation;
            }
            else if (currentCharacter == '|')
            {
                currentTokenizationType = SqlTokenizationType.SinglePipe;
            }
            else if (IsCompoundableOperatorCharacter(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.SingleOtherCompoundableOperator;
                currentNodeValue.Append(currentCharacter);
            }
            else if (IsOperatorCharacter(currentCharacter))
            {
                tokenContainer.Add(new Token(SqlTokenType.OtherOperator, currentCharacter.ToString()));
            }
            else
            {
                currentTokenizationType = SqlTokenizationType.OtherNode;
                currentNodeValue.Append(currentCharacter);
            }
        }
Esempio n. 27
0
        public string FormatSQLTokens(ITokenList sqlTokenList)
        {
            StringBuilder outString = new StringBuilder();

            if (sqlTokenList.HasUnfinishedToken)
            {
                outString.Append(ErrorOutputPrefix);
            }

            foreach (var entry in sqlTokenList)
            {
                switch (entry.Type)
                {
                case SqlTokenType.MultiLineComment:
                    outString.Append("/*");
                    outString.Append(entry.Value);
                    outString.Append("*/");
                    break;

                case SqlTokenType.SingleLineComment:
                    outString.Append("--");
                    outString.Append(entry.Value);
                    break;

                case SqlTokenType.SingleLineCommentCStyle:
                    outString.Append("//");
                    outString.Append(entry.Value);
                    break;

                case SqlTokenType.String:
                    outString.Append("'");
                    outString.Append(entry.Value.Replace("'", "''"));
                    outString.Append("'");
                    break;

                case SqlTokenType.NationalString:
                    outString.Append("N'");
                    outString.Append(entry.Value.Replace("'", "''"));
                    outString.Append("'");
                    break;

                case SqlTokenType.QuotedString:
                    outString.Append("\"");
                    outString.Append(entry.Value.Replace("\"", "\"\""));
                    outString.Append("\"");
                    break;

                case SqlTokenType.BracketQuotedName:
                    outString.Append("[");
                    outString.Append(entry.Value.Replace("]", "]]"));
                    outString.Append("]");
                    break;

                case SqlTokenType.OpenParens:
                case SqlTokenType.CloseParens:
                case SqlTokenType.Comma:
                case SqlTokenType.Period:
                case SqlTokenType.Semicolon:
                case SqlTokenType.Colon:
                case SqlTokenType.Asterisk:
                case SqlTokenType.EqualsSign:
                case SqlTokenType.OtherNode:
                case SqlTokenType.WhiteSpace:
                case SqlTokenType.OtherOperator:
                case SqlTokenType.Number:
                case SqlTokenType.BinaryValue:
                case SqlTokenType.MonetaryValue:
                case SqlTokenType.PseudoName:
                    outString.Append(entry.Value);
                    break;

                default:
                    throw new Exception("Unrecognized Token Type in Token List!");
                }
            }

            return(outString.ToString());
        }
Esempio n. 28
0
 public void ResetClassList()
 {
     classList = null;
 }
Esempio n. 29
0
        private static void ProcessOrOpenToken(ref SqlTokenizationType?currentTokenizationType, StringBuilder currentNodeValue, char currentCharacter, ITokenList tokenContainer)
        {
            if (currentTokenizationType != null)
            {
                throw new Exception("Cannot start a new Token: existing Tokenization Type is not null");
            }

            //start a new value.
            currentNodeValue.Length = 0;

            if (IsWhitespace(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.WhiteSpace;
                currentNodeValue.Append(currentCharacter);
            }
            else if (currentCharacter == '-')
            {
                currentTokenizationType = SqlTokenizationType.SingleHyphen;
            }
            else if (currentCharacter == '$')
            {
                currentTokenizationType = SqlTokenizationType.SingleDollar;
            }
            else if (currentCharacter == '/')
            {
                currentTokenizationType = SqlTokenizationType.SingleSlash;
            }
            else if (currentCharacter == 'N')
            {
                currentTokenizationType = SqlTokenizationType.SingleN;
            }
            else if (currentCharacter == '\'')
            {
                currentTokenizationType = SqlTokenizationType.String;
            }
            else if (currentCharacter == '"')
            {
                currentTokenizationType = SqlTokenizationType.QuotedString;
            }
            else if (currentCharacter == '[')
            {
                currentTokenizationType = SqlTokenizationType.BracketQuotedName;
            }
            else if (currentCharacter == '(')
            {
                tokenContainer.Add(new Token(SqlTokenType.OpenParens, currentCharacter.ToString()));
            }
            else if (currentCharacter == ')')
            {
                tokenContainer.Add(new Token(SqlTokenType.CloseParens, currentCharacter.ToString()));
            }
            else if (currentCharacter == ',')
            {
                tokenContainer.Add(new Token(SqlTokenType.Comma, currentCharacter.ToString()));
            }
            else if (currentCharacter == '.')
            {
                currentTokenizationType = SqlTokenizationType.SinglePeriod;
            }
            else if (currentCharacter == '0')
            {
                currentTokenizationType = SqlTokenizationType.SingleZero;
            }
            else if (currentCharacter >= '1' && currentCharacter <= '9')
            {
                currentTokenizationType = SqlTokenizationType.Number;
                currentNodeValue.Append(currentCharacter);
            }
            else if (IsCurrencyPrefix(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.MonetaryValue;
                currentNodeValue.Append(currentCharacter);
            }
            else if (currentCharacter == ';')
            {
                tokenContainer.Add(new Token(SqlTokenType.Semicolon, currentCharacter.ToString()));
            }
            else if (currentCharacter == ':')
            {
                tokenContainer.Add(new Token(SqlTokenType.Colon, currentCharacter.ToString()));
            }
            else if (currentCharacter == '*')
            {
                tokenContainer.Add(new Token(SqlTokenType.Asterisk, currentCharacter.ToString()));
            }
            else if (currentCharacter == '=')
            {
                tokenContainer.Add(new Token(SqlTokenType.EqualsSign, currentCharacter.ToString()));
            }
            else if (currentCharacter == '<')
            {
                currentTokenizationType = SqlTokenizationType.SingleLT;
            }
            else if (currentCharacter == '!')
            {
                currentTokenizationType = SqlTokenizationType.SingleExclamation;
            }
            else if (currentCharacter == '|')
            {
                currentTokenizationType = SqlTokenizationType.SinglePipe;
            }
            else if (IsCompoundableOperatorCharacter(currentCharacter))
            {
                currentTokenizationType = SqlTokenizationType.SingleOtherCompoundableOperator;
                currentNodeValue.Append(currentCharacter);
            }
            else if (IsOperatorCharacter(currentCharacter))
            {
                tokenContainer.Add(new Token(SqlTokenType.OtherOperator, currentCharacter.ToString()));
            }
            else
            {
                currentTokenizationType = SqlTokenizationType.OtherNode;
                currentNodeValue.Append(currentCharacter);
            }
        }
 private void ProcessCompoundKeywordWithError(ITokenList tokenList, ParseTree sqlTree, XmlElement currentContainerElement, ref int tokenID, List<int> significantTokenPositions, int keywordCount)
 {
     ProcessCompoundKeyword(tokenList, sqlTree, currentContainerElement, ref tokenID, significantTokenPositions, keywordCount);
     sqlTree.ErrorFound = true;
 }
 private void ProcessCompoundKeyword(ITokenList tokenList, ParseTree sqlTree, XmlElement targetContainer, ref int tokenID, List<int> significantTokenPositions, int keywordCount)
 {
     XmlElement compoundKeyword = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_COMPOUNDKEYWORD, "", targetContainer);
     string targetText = ExtractTokensString(tokenList, significantTokenPositions.GetRange(0, keywordCount)).TrimEnd();
     compoundKeyword.SetAttribute(SqlXmlConstants.ANAME_SIMPLETEXT, targetText);
     AppendNodesWithMapping(sqlTree, tokenList.GetRangeByIndex(significantTokenPositions[0], significantTokenPositions[keywordCount - 1]), SqlXmlConstants.ENAME_OTHERKEYWORD, compoundKeyword);
     tokenID = significantTokenPositions[keywordCount - 1];
 }
 private bool IsFollowedByLineBreakingWhiteSpaceOrSingleLineCommentOrEnd(ITokenList tokenList, int tokenID)
 {
     int currTokenID = tokenID + 1;
     while (tokenList.Count >= currTokenID + 1)
     {
         if (tokenList[currTokenID].Type == SqlTokenType.SingleLineComment)
             return true;
         else if (tokenList[currTokenID].Type == SqlTokenType.WhiteSpace)
         {
             if (Regex.IsMatch(tokenList[currTokenID].Value, @"(\r|\n)+"))
                 return true;
             else
                 currTokenID++;
         }
         else
             return false;
     }
     return true;
 }
        private List<int> GetSignificantTokenPositions(ITokenList tokenList, int tokenID, int searchDistance)
        {
            List<int> significantTokenPositions = new List<int>();
            int originalTokenID = tokenID;

            while (tokenID < tokenList.Count && significantTokenPositions.Count < searchDistance)
            {
                if (tokenList[tokenID].Type == SqlTokenType.OtherNode
                    || tokenList[tokenID].Type == SqlTokenType.BracketQuotedName
                    || tokenList[tokenID].Type == SqlTokenType.Comma
                    )
                {
                    significantTokenPositions.Add(tokenID);
                    tokenID++;

                    //found a possible phrase component - skip past any upcoming whitespace or comments, keeping track.
                    while (tokenID < tokenList.Count
                        && (tokenList[tokenID].Type == SqlTokenType.WhiteSpace
                            || tokenList[tokenID].Type == SqlTokenType.SingleLineComment
                            || tokenList[tokenID].Type == SqlTokenType.MultiLineComment
                            )
                        )
                    {
                        tokenID++;
                    }
                }
                else
                    //we're not interested in any other node types
                    break;
            }

            return significantTokenPositions;
        }
        private string GetKeywordMatchPhrase(ITokenList tokenList, int tokenID, ref List<string> rawKeywordParts, ref List<int> tokenCounts, ref List<List<IToken>> overflowNodes)
        {
            string phrase = "";
            int phraseComponentsFound = 0;
            rawKeywordParts = new List<string>();
            overflowNodes = new List<List<IToken>>();
            tokenCounts = new List<int>();
            string precedingWhitespace = "";
            int originalTokenID = tokenID;

            while (tokenID < tokenList.Count && phraseComponentsFound < 7)
            {
                if (tokenList[tokenID].Type == SqlTokenType.OtherNode
                    || tokenList[tokenID].Type == SqlTokenType.BracketQuotedName
                    || tokenList[tokenID].Type == SqlTokenType.Comma
                    )
                {
                    phrase += tokenList[tokenID].Value.ToUpperInvariant() + " ";
                    phraseComponentsFound++;
                    rawKeywordParts.Add(precedingWhitespace + tokenList[tokenID].Value);

                    tokenID++;
                    tokenCounts.Add(tokenID - originalTokenID);

                    //found a possible phrase component - skip past any upcoming whitespace or comments, keeping track.
                    overflowNodes.Add(new List<IToken>());
                    precedingWhitespace = "";
                    while (tokenID < tokenList.Count
                        && (tokenList[tokenID].Type == SqlTokenType.WhiteSpace
                            || tokenList[tokenID].Type == SqlTokenType.SingleLineComment
                            || tokenList[tokenID].Type == SqlTokenType.MultiLineComment
                            )
                        )
                    {
                        if (tokenList[tokenID].Type == SqlTokenType.WhiteSpace)
                            precedingWhitespace += tokenList[tokenID].Value;
                        else
                            overflowNodes[phraseComponentsFound-1].Add(tokenList[tokenID]);

                        tokenID++;
                    }
                }
                else
                    //we're not interested in any other node types
                    break;
            }

            return phrase;
        }
        public static IList <string> GetPartialWordMatch(string wordToPartiallyMatch, ITokenList tokenList)
        {
            var strings = new List <string>();

            var rx = new Regex(@$ "{wordToPartiallyMatch}.");

            foreach (var x in tokenList)
            {
                var matches = rx.Matches(x);
                if (matches != null && matches.Count > 0)
                {
                    strings.Add(x);
                }
            }

            return(strings);
        }
        public XmlDocument ParseSQL(ITokenList tokenList)
        {
            ParseTree sqlTree = new ParseTree(SqlXmlConstants.ENAME_SQL_ROOT);
            sqlTree.ErrorFound = tokenList.HasErrors;
            sqlTree.StartNewStatement();

            int tokenCount = tokenList.Count;
            int tokenID = 0;
            while (tokenID < tokenCount)
            {
                IToken token = tokenList[tokenID];

                switch (token.Type)
                {
                    case SqlTokenType.OpenParens:
                        XmlElement firstNonCommentParensSibling = sqlTree.GetFirstNonWhitespaceNonCommentChildElement(sqlTree.CurrentContainer);
                        bool isInsertOrValuesClause = (
                            firstNonCommentParensSibling != null
                            && (
                                (firstNonCommentParensSibling.Name.Equals(SqlXmlConstants.ENAME_OTHERKEYWORD)
                                   && firstNonCommentParensSibling.InnerText.ToUpperInvariant().StartsWith("INSERT")
                                   )
                                ||
                                (firstNonCommentParensSibling.Name.Equals(SqlXmlConstants.ENAME_COMPOUNDKEYWORD)
                                   && firstNonCommentParensSibling.GetAttribute(SqlXmlConstants.ANAME_SIMPLETEXT).ToUpperInvariant().StartsWith("INSERT ")
                                   )
                                ||
                                (firstNonCommentParensSibling.Name.Equals(SqlXmlConstants.ENAME_OTHERKEYWORD)
                                   && firstNonCommentParensSibling.InnerText.ToUpperInvariant().StartsWith("VALUES")
                                   )
                               )
                            );

                        if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_CTE_ALIAS)
                            && sqlTree.CurrentContainer.ParentNode.Name.Equals(SqlXmlConstants.ENAME_CTE_WITH_CLAUSE)
                            )
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_PARENS, "");
                        else if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                            && sqlTree.CurrentContainer.ParentNode.Name.Equals(SqlXmlConstants.ENAME_CTE_AS_BLOCK)
                            )
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS, "");
                        else if (firstNonCommentParensSibling == null
                            && sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_SELECTIONTARGET)
                            )
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS, "");
                        else if (firstNonCommentParensSibling != null
                            && firstNonCommentParensSibling.Name.Equals(SqlXmlConstants.ENAME_SET_OPERATOR_CLAUSE)
                            )
                        {
                            sqlTree.ConsiderStartingNewClause();
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS, "");
                        }
                        else if (IsLatestTokenADDLDetailValue(sqlTree))
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDLDETAIL_PARENS, "");
                        else if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                            || sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_DDL_OTHER_BLOCK)
                            || isInsertOrValuesClause
                            )
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_PARENS, "");
                        else if (IsLatestTokenAMiscName(sqlTree.CurrentContainer))
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_FUNCTION_PARENS, "");
                        else
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_EXPRESSION_PARENS, "");
                        break;

                    case SqlTokenType.CloseParens:
                        //we're not likely to actually have a "SingleStatement" in parens, but
                        // we definitely want the side-effects (all the lower-level escapes)
                        sqlTree.EscapeAnySingleOrPartialStatementContainers();

                        //check whether we expected to end the parens...
                        if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_DDLDETAIL_PARENS)
                            || sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_DDL_PARENS)
                            || sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_FUNCTION_PARENS)
                            || sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_EXPRESSION_PARENS)
                            || sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS)
                            )
                        {
                            sqlTree.MoveToAncestorContainer(1); //unspecified parent node...
                        }
                        else if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.CurrentContainer.ParentNode.Name.Equals(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS)
                                && sqlTree.CurrentContainer.ParentNode.ParentNode.Name.Equals(SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.CurrentContainer.ParentNode.ParentNode.ParentNode.Name.Equals(SqlXmlConstants.ENAME_CTE_AS_BLOCK)
                                )
                        {
                            sqlTree.MoveToAncestorContainer(4, SqlXmlConstants.ENAME_CTE_WITH_CLAUSE);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT, "");
                        }
                        else if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && (
                                    sqlTree.CurrentContainer.ParentNode.Name.Equals(SqlXmlConstants.ENAME_EXPRESSION_PARENS)
                                    || sqlTree.CurrentContainer.ParentNode.Name.Equals(SqlXmlConstants.ENAME_SELECTIONTARGET_PARENS)
                                )
                            )
                        {
                            sqlTree.MoveToAncestorContainer(2); //unspecified grandfather node.
                        }
                        else
                        {
                            sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERNODE, ")");
                        }
                        break;

                    case SqlTokenType.OtherNode:

                        //prepare multi-keyword detection by "peeking" up to 7 keywords ahead
                        List<int> significantTokenPositions = GetSignificantTokenPositions(tokenList, tokenID, 7);
                        string significantTokensString = ExtractTokensString(tokenList, significantTokenPositions);

                        if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_PERMISSIONS_DETAIL))
                        {
                            //if we're in a permissions detail clause, we can expect all sorts of statements
                            // starters and should ignore them all; the only possible keywords to escape are
                            // "ON" and "TO".
                            if (significantTokensString.StartsWith("ON "))
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_PERMISSIONS_TARGET, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if (significantTokensString.StartsWith("TO ")
                                || significantTokensString.StartsWith("FROM ")
                                )
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_PERMISSIONS_RECIPIENT, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                //default to "some classification of permission"
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("CREATE PROC")
                            || significantTokensString.StartsWith("CREATE FUNC")
                            || significantTokensString.StartsWith("CREATE TRIGGER ")
                            || significantTokensString.StartsWith("CREATE VIEW ")
                            || significantTokensString.StartsWith("ALTER PROC")
                            || significantTokensString.StartsWith("ALTER FUNC")
                            || significantTokensString.StartsWith("ALTER TRIGGER ")
                            || significantTokensString.StartsWith("ALTER VIEW ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK, "");
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (_CursorDetector.IsMatch(significantTokensString))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CURSOR_DECLARATION, "");
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                            && _TriggerConditionDetector.IsMatch(significantTokensString)
                            )
                        {
                            //horrible complicated forward-search, to avoid having to keep a different "Trigger Condition" state for Update, Insert and Delete statement-starting keywords
                            Match triggerConditions = _TriggerConditionDetector.Match(significantTokensString);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_TRIGGER_CONDITION, "");
                            XmlElement triggerConditionType = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_COMPOUNDKEYWORD, "");

                            //first set the "trigger condition type": FOR, INSTEAD OF, AFTER
                            string triggerConditionTypeSimpleText = triggerConditions.Groups[1].Value;
                            triggerConditionType.SetAttribute(SqlXmlConstants.ANAME_SIMPLETEXT, triggerConditionTypeSimpleText);
                            int triggerConditionTypeNodeCount = triggerConditionTypeSimpleText.Split(new char[] { ' ' }).Length; //there's probably a better way of counting words...
                            AppendNodesWithMapping(sqlTree, tokenList.GetRangeByIndex(significantTokenPositions[0], significantTokenPositions[triggerConditionTypeNodeCount - 1]), SqlXmlConstants.ENAME_OTHERKEYWORD, triggerConditionType);

                            //then get the count of conditions (INSERT, UPDATE, DELETE) and add those too...
                            int triggerConditionNodeCount = triggerConditions.Groups[2].Value.Split(new char[] { ' ' }).Length - 2; //there's probably a better way of counting words...
                            AppendNodesWithMapping(sqlTree, tokenList.GetRangeByIndex(significantTokenPositions[triggerConditionTypeNodeCount - 1] + 1, significantTokenPositions[triggerConditionTypeNodeCount + triggerConditionNodeCount - 1]), SqlXmlConstants.ENAME_OTHERKEYWORD, sqlTree.CurrentContainer);
                            tokenID = significantTokenPositions[triggerConditionTypeNodeCount + triggerConditionNodeCount - 1];
                            sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK);
                        }
                        else if (significantTokensString.StartsWith("FOR "))
                        {
                            sqlTree.EscapeAnyBetweenConditions();
                            sqlTree.EscapeAnySelectionTarget();
                            sqlTree.EscapeJoinCondition();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CURSOR_DECLARATION))
                            {
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CURSOR_FOR_BLOCK, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                                sqlTree.StartNewStatement();
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(3, SqlXmlConstants.ENAME_CURSOR_FOR_BLOCK)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(4, SqlXmlConstants.ENAME_CURSOR_DECLARATION);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CURSOR_FOR_OPTIONS, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                //Assume FOR clause if we're at clause level
                                // (otherwise, eg in OPTIMIZE FOR UNKNOWN, this will just not do anything)
                                sqlTree.ConsiderStartingNewClause();

                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("CREATE ")
                            || significantTokensString.StartsWith("ALTER ")
                            || significantTokensString.StartsWith("DECLARE ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_OTHER_BLOCK, "");
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("GRANT ")
                            || significantTokensString.StartsWith("DENY ")
                            || significantTokensString.StartsWith("REVOKE ")
                            )
                        {
                            if (significantTokensString.StartsWith("GRANT ")
                                && sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_DDL_WITH_CLAUSE)
                                && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK)
                                && sqlTree.GetFirstNonWhitespaceNonCommentChildElement(sqlTree.CurrentContainer) == null
                                )
                            {
                                //this MUST be a "WITH GRANT OPTION" option...
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                            else
                            {
                                sqlTree.ConsiderStartingNewStatement();
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_PERMISSIONS_BLOCK, token.Value, SqlXmlConstants.ENAME_PERMISSIONS_DETAIL);
                            }
                        }
                        else if (sqlTree.CurrentContainer.Name.Equals(SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                            && significantTokensString.StartsWith("RETURNS ")
                            )
                        {
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_RETURNS, ""));
                        }
                        else if (significantTokensString.StartsWith("AS "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK))
                            {
                                KeywordType nextKeywordType;
                                bool isDataTypeDefinition = false;
                                if (significantTokenPositions.Count > 1
                                    && KeywordList.TryGetValue(tokenList[significantTokenPositions[1]].Value, out nextKeywordType)
                                    )
                                    if (nextKeywordType == KeywordType.DataTypeKeyword)
                                        isDataTypeDefinition = true;

                                if (isDataTypeDefinition)
                                {
                                    //this is actually a data type declaration (redundant "AS"...), save as regular token.
                                    sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                }
                                else
                                {
                                    //this is the start of the object content definition
                                    sqlTree.StartNewContainer(SqlXmlConstants.ENAME_DDL_AS_BLOCK, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                                    sqlTree.StartNewStatement();
                                }
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_DDL_WITH_CLAUSE)
                                && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_DDL_AS_BLOCK, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                                sqlTree.StartNewStatement();
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CTE_ALIAS)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CTE_WITH_CLAUSE)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_CTE_WITH_CLAUSE);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CTE_AS_BLOCK, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("BEGIN DISTRIBUTED TRANSACTION ")
                            || significantTokensString.StartsWith("BEGIN DISTRIBUTED TRAN ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BEGIN_TRANSACTION, ""), ref tokenID, significantTokenPositions, 3);
                        }
                        else if (significantTokensString.StartsWith("BEGIN TRANSACTION ")
                            || significantTokensString.StartsWith("BEGIN TRAN ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BEGIN_TRANSACTION, ""), ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("SAVE TRANSACTION ")
                            || significantTokensString.StartsWith("SAVE TRAN ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SAVE_TRANSACTION, ""), ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("COMMIT TRANSACTION ")
                            || significantTokensString.StartsWith("COMMIT TRAN ")
                            || significantTokensString.StartsWith("COMMIT WORK ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_COMMIT_TRANSACTION, ""), ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("COMMIT "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_COMMIT_TRANSACTION, token.Value));
                        }
                        else if (significantTokensString.StartsWith("ROLLBACK TRANSACTION ")
                            || significantTokensString.StartsWith("ROLLBACK TRAN ")
                            || significantTokensString.StartsWith("ROLLBACK WORK ")
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_ROLLBACK_TRANSACTION, ""), ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("ROLLBACK "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_ROLLBACK_TRANSACTION, token.Value));
                        }
                        else if (significantTokensString.StartsWith("BEGIN TRY "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            XmlElement newTryBlock = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_TRY_BLOCK, "");
                            XmlElement tryContainerOpen = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_OPEN, "", newTryBlock);
                            ProcessCompoundKeyword(tokenList, sqlTree, tryContainerOpen, ref tokenID, significantTokenPositions, 2);
                            XmlElement tryMultiContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT, "", newTryBlock);
                            sqlTree.StartNewStatement(tryMultiContainer);
                        }
                        else if (significantTokensString.StartsWith("BEGIN CATCH "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            XmlElement newCatchBlock = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CATCH_BLOCK, "");
                            XmlElement catchContainerOpen = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_OPEN, "", newCatchBlock);
                            ProcessCompoundKeyword(tokenList, sqlTree, catchContainerOpen, ref tokenID, significantTokenPositions, 2);
                            XmlElement catchMultiContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT, "", newCatchBlock);
                            sqlTree.StartNewStatement(catchMultiContainer);
                        }
                        else if (significantTokensString.StartsWith("BEGIN "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.StartNewContainer(SqlXmlConstants.ENAME_BEGIN_END_BLOCK, token.Value, SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT);
                            sqlTree.StartNewStatement();
                        }
                        else if (significantTokensString.StartsWith("MERGE "))
                        {
                            //According to BOL, MERGE is a fully reserved keyword from compat 100 onwards, for the MERGE statement only.
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.ConsiderStartingNewClause();
                            sqlTree.StartNewContainer(SqlXmlConstants.ENAME_MERGE_CLAUSE, token.Value, SqlXmlConstants.ENAME_MERGE_TARGET);
                        }
                        else if (significantTokensString.StartsWith("USING "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_MERGE_TARGET))
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_MERGE_CLAUSE);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_MERGE_USING, token.Value, SqlXmlConstants.ENAME_SELECTIONTARGET);
                            }
                            else
                                sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERNODE, token.Value);
                        }
                        else if (significantTokensString.StartsWith("ON "))
                        {
                            sqlTree.EscapeAnySelectionTarget();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_MERGE_USING))
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_MERGE_CLAUSE);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_MERGE_CONDITION, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if (!sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                                && !sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_OTHER_BLOCK)
                                && !sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_DDL_WITH_CLAUSE)
                                && !sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_EXPRESSION_PARENS)
                                && !ContentStartsWithKeyword(sqlTree.CurrentContainer, "SET")
                                )
                            {
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_JOIN_ON_SECTION, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("CASE "))
                        {
                            sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CASE_STATEMENT, token.Value, SqlXmlConstants.ENAME_CASE_INPUT);
                        }
                        else if (significantTokensString.StartsWith("WHEN "))
                        {
                            sqlTree.EscapeMergeAction();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CASE_INPUT)
                                || (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                    && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CASE_THEN)
                                    )
                                )
                            {
                                if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CASE_INPUT))
                                    sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_CASE_STATEMENT);
                                else
                                    sqlTree.MoveToAncestorContainer(3, SqlXmlConstants.ENAME_CASE_STATEMENT);

                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CASE_WHEN, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if ((sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                    && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_MERGE_CONDITION)
                                    )
                                || sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_MERGE_WHEN)
                                )
                            {
                                if (sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_MERGE_CONDITION))
                                    sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_MERGE_CLAUSE);
                                else
                                    sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_MERGE_CLAUSE);

                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_MERGE_WHEN, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                                sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERNODE, token.Value);
                        }
                        else if (significantTokensString.StartsWith("THEN "))
                        {
                            sqlTree.EscapeAnyBetweenConditions();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CASE_WHEN)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_CASE_WHEN);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CASE_THEN, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_MERGE_WHEN)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_MERGE_WHEN);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_MERGE_THEN, token.Value, SqlXmlConstants.ENAME_MERGE_ACTION);
                                sqlTree.StartNewStatement();
                            }
                            else
                                sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERNODE, token.Value);
                        }
                        else if (significantTokensString.StartsWith("OUTPUT "))
                        {
                            bool isSprocArgument = false;

                            //We're looking for sproc calls - they can't be nested inside anything else (as far as I know)
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && (ContentStartsWithKeyword(sqlTree.CurrentContainer, "EXEC")
                                    || ContentStartsWithKeyword(sqlTree.CurrentContainer, "EXECUTE")
                                    || ContentStartsWithKeyword(sqlTree.CurrentContainer, null)
                                    )
                                )
                            {
                                isSprocArgument = true;
                            }

                            if (!isSprocArgument)
                            {
                                sqlTree.EscapeMergeAction();
                                sqlTree.ConsiderStartingNewClause();
                            }

                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("OPTION "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_DDL_WITH_CLAUSE)
                                )
                            {
                                //"OPTION" keyword here is NOT indicative of a new clause.
                            }
                            else
                            {
                                sqlTree.EscapeMergeAction();
                                sqlTree.ConsiderStartingNewClause();
                            }
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("END TRY "))
                        {
                            sqlTree.EscapeAnySingleOrPartialStatementContainers();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT)
                                && sqlTree.PathNameMatches(3, SqlXmlConstants.ENAME_TRY_BLOCK)
                                )
                            {
                                //clause.statement.multicontainer.try
                                XmlElement tryBlock = (XmlElement)sqlTree.CurrentContainer.ParentNode.ParentNode.ParentNode;
                                XmlElement tryContainerClose = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, "", tryBlock);
                                ProcessCompoundKeyword(tokenList, sqlTree, tryContainerClose, ref tokenID, significantTokenPositions, 2);
                                sqlTree.CurrentContainer = (XmlElement)tryBlock.ParentNode;
                            }
                            else
                            {
                                ProcessCompoundKeywordWithError(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                            }
                        }
                        else if (significantTokensString.StartsWith("END CATCH "))
                        {
                            sqlTree.EscapeAnySingleOrPartialStatementContainers();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT)
                                && sqlTree.PathNameMatches(3, SqlXmlConstants.ENAME_CATCH_BLOCK)
                                )
                            {
                                //clause.statement.multicontainer.catch
                                XmlElement catchBlock = (XmlElement)sqlTree.CurrentContainer.ParentNode.ParentNode.ParentNode;
                                XmlElement catchContainerClose = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, "", catchBlock);
                                ProcessCompoundKeyword(tokenList, sqlTree, catchContainerClose, ref tokenID, significantTokenPositions, 2);
                                sqlTree.CurrentContainer = (XmlElement)catchBlock.ParentNode;
                            }
                            else
                            {
                                ProcessCompoundKeywordWithError(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                            }
                        }
                        else if (significantTokensString.StartsWith("END "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CASE_THEN)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(3, SqlXmlConstants.ENAME_CASE_STATEMENT);
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, ""));
                                sqlTree.MoveToAncestorContainer(1); //unnamed container
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CASE_ELSE)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_CASE_STATEMENT);
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, ""));
                                sqlTree.MoveToAncestorContainer(1); //unnamed container
                            }
                            else
                            {
                                //Begin/End block handling
                                sqlTree.EscapeAnySingleOrPartialStatementContainers();

                                if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                    && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                    && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_CONTAINER_MULTISTATEMENT)
                                    && sqlTree.PathNameMatches(3, SqlXmlConstants.ENAME_BEGIN_END_BLOCK)
                                    )
                                {
                                    XmlElement beginBlock = (XmlElement)sqlTree.CurrentContainer.ParentNode.ParentNode.ParentNode;
                                    XmlElement beginContainerClose = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, "", beginBlock);
                                    sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, beginContainerClose);
                                    sqlTree.CurrentContainer = (XmlElement)beginBlock.ParentNode;
                                }
                                else
                                {
                                    sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                }
                            }
                        }
                        else if (significantTokensString.StartsWith("GO "))
                        {
                            sqlTree.EscapeAnySingleOrPartialStatementContainers();

                            if ((tokenID == 0 || IsLineBreakingWhiteSpaceOrComment(tokenList[tokenID - 1]))
                                && IsFollowedByLineBreakingWhiteSpaceOrSingleLineCommentOrEnd(tokenList, tokenID)
                                )
                            {
                                // we found a batch separator - were we supposed to?
                                if (sqlTree.FindValidBatchEnd())
                                {
                                    XmlElement sqlRoot = sqlTree.DocumentElement;
                                    XmlElement batchSeparator = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BATCH_SEPARATOR, "", sqlRoot);
                                    sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, batchSeparator);
                                    sqlTree.StartNewStatement(sqlRoot);
                                }
                                else
                                {
                                    sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                }
                            }
                            else
                            {
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("EXECUTE AS "))
                        {
                            bool executeAsInWithOptions = false;
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_DDL_WITH_CLAUSE)
                                && (IsLatestTokenAComma(sqlTree)
                                    || !sqlTree.HasNonWhiteSpaceNonCommentContent(sqlTree.CurrentContainer)
                                    )
                                )
                                executeAsInWithOptions = true;

                            if (!executeAsInWithOptions)
                            {
                                sqlTree.ConsiderStartingNewStatement();
                                sqlTree.ConsiderStartingNewClause();
                            }

                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("EXEC ")
                            || significantTokensString.StartsWith("EXECUTE ")
                            )
                        {
                            bool execShouldntTryToStartNewStatement = false;

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && (ContentStartsWithKeyword(sqlTree.CurrentContainer, "INSERT")
                                    || ContentStartsWithKeyword(sqlTree.CurrentContainer, "INSERT INTO")
                                    )
                                )
                            {
                                int existingClauseCount = sqlTree.CurrentContainer.SelectNodes(string.Format("../{0}", SqlXmlConstants.ENAME_SQL_CLAUSE)).Count;
                                if (existingClauseCount == 1)
                                    execShouldntTryToStartNewStatement = true;
                            }

                            if (!execShouldntTryToStartNewStatement)
                                sqlTree.ConsiderStartingNewStatement();

                            sqlTree.ConsiderStartingNewClause();

                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (_JoinDetector.IsMatch(significantTokensString))
                        {
                            sqlTree.ConsiderStartingNewClause();
                            string joinText = _JoinDetector.Match(significantTokensString).Value;
                            int targetKeywordCount = joinText.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries).Length;
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, targetKeywordCount);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SELECTIONTARGET, "");
                        }
                        else if (significantTokensString.StartsWith("UNION ALL "))
                        {
                            sqlTree.ConsiderStartingNewClause();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SET_OPERATOR_CLAUSE, ""), ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("UNION ")
                            || significantTokensString.StartsWith("INTERSECT ")
                            || significantTokensString.StartsWith("EXCEPT ")
                            )
                        {
                            sqlTree.ConsiderStartingNewClause();
                            XmlElement unionClause = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SET_OPERATOR_CLAUSE, "");
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, unionClause);
                        }
                        else if (significantTokensString.StartsWith("WHILE "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            XmlElement newWhileLoop = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_WHILE_LOOP, "");
                            XmlElement whileContainerOpen = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_OPEN, "", newWhileLoop);
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, whileContainerOpen);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BOOLEAN_EXPRESSION, "", newWhileLoop);
                        }
                        else if (significantTokensString.StartsWith("IF "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.StartNewContainer(SqlXmlConstants.ENAME_IF_STATEMENT, token.Value, SqlXmlConstants.ENAME_BOOLEAN_EXPRESSION);
                        }
                        else if (significantTokensString.StartsWith("ELSE "))
                        {
                            sqlTree.EscapeAnyBetweenConditions();
                            sqlTree.EscapeAnySelectionTarget();
                            sqlTree.EscapeJoinCondition();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CASE_THEN)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(3, SqlXmlConstants.ENAME_CASE_STATEMENT);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_CASE_ELSE, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                sqlTree.EscapePartialStatementContainers();

                                if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                    && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                    && sqlTree.PathNameMatches(2, SqlXmlConstants.ENAME_CONTAINER_SINGLESTATEMENT)
                                    )
                                {
                                    //we need to pop up the single-statement containers stack to the next "if" that doesn't have an "else" (if any; else error).
                                    // LOCAL SEARCH - we're not actually changing the "CurrentContainer" until we decide to start a statement.
                                    XmlElement currentNode = (XmlElement)sqlTree.CurrentContainer.ParentNode.ParentNode;
                                    bool stopSearching = false;
                                    while (!stopSearching)
                                    {
                                        if (sqlTree.PathNameMatches(currentNode, 1, SqlXmlConstants.ENAME_IF_STATEMENT))
                                        {
                                            //if this is in an "If", then the "Else" must still be available - yay!
                                            sqlTree.CurrentContainer = (XmlElement)currentNode.ParentNode;
                                            sqlTree.StartNewContainer(SqlXmlConstants.ENAME_ELSE_CLAUSE, token.Value, SqlXmlConstants.ENAME_CONTAINER_SINGLESTATEMENT);
                                            sqlTree.StartNewStatement();
                                            stopSearching = true;
                                        }
                                        else if (sqlTree.PathNameMatches(currentNode, 1, SqlXmlConstants.ENAME_ELSE_CLAUSE))
                                        {
                                            //If this is in an "Else", we should skip its parent "IF" altogether, and go to the next singlestatementcontainer candidate.
                                            //singlestatementcontainer.else.if.clause.statement.NEWCANDIDATE
                                            currentNode = (XmlElement)currentNode.ParentNode.ParentNode.ParentNode.ParentNode.ParentNode;
                                        }
                                        else if (sqlTree.PathNameMatches(currentNode, 1, SqlXmlConstants.ENAME_WHILE_LOOP))
                                        {
                                            //If this is in a "While", we should skip to the next singlestatementcontainer candidate.
                                            //singlestatementcontainer.while.clause.statement.NEWCANDIDATE
                                            currentNode = (XmlElement)currentNode.ParentNode.ParentNode.ParentNode.ParentNode;
                                        }
                                        else
                                        {
                                            //if this isn't a known single-statement container, then we're lost.
                                            sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                            stopSearching = true;
                                        }
                                    }
                                }
                                else
                                {
                                    sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                }
                            }
                        }
                        else if (significantTokensString.StartsWith("INSERT INTO "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.ConsiderStartingNewClause();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("NATIONAL CHARACTER VARYING "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 3);
                        }
                        else if (significantTokensString.StartsWith("NATIONAL CHAR VARYING "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 3);
                        }
                        else if (significantTokensString.StartsWith("BINARY VARYING "))
                        {
                            //TODO: Figure out how to handle "Compound Keyword Datatypes" so they are still correctly highlighted
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("CHAR VARYING "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("CHARACTER VARYING "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("DOUBLE PRECISION "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("NATIONAL CHARACTER "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("NATIONAL CHAR "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("NATIONAL TEXT "))
                        {
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("INSERT "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.ConsiderStartingNewClause();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("BULK INSERT "))
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.ConsiderStartingNewClause();
                            ProcessCompoundKeyword(tokenList, sqlTree, sqlTree.CurrentContainer, ref tokenID, significantTokenPositions, 2);
                        }
                        else if (significantTokensString.StartsWith("SELECT "))
                        {
                            if (sqlTree.NewStatementDue)
                                sqlTree.ConsiderStartingNewStatement();

                            bool selectShouldntTryToStartNewStatement = false;

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE))
                            {
                                XmlElement firstStatementClause = sqlTree.GetFirstNonWhitespaceNonCommentChildElement(sqlTree.CurrentContainer.ParentNode);

                                bool isPrecededByInsertStatement = false;
                                foreach (XmlElement clause in sqlTree.CurrentContainer.ParentNode.SelectNodes(SqlXmlConstants.ENAME_SQL_CLAUSE))
                                    if (ContentStartsWithKeyword(clause, "INSERT"))
                                        isPrecededByInsertStatement = true;

                                if (isPrecededByInsertStatement)
                                {
                                    bool existingSelectClauseFound = false;
                                    foreach (XmlElement clause in sqlTree.CurrentContainer.ParentNode.SelectNodes(SqlXmlConstants.ENAME_SQL_CLAUSE))
                                        if (ContentStartsWithKeyword(clause, "SELECT"))
                                            existingSelectClauseFound = true;

                                    bool existingValuesClauseFound = false;
                                    foreach (XmlElement clause in sqlTree.CurrentContainer.ParentNode.SelectNodes(SqlXmlConstants.ENAME_SQL_CLAUSE))
                                        if (ContentStartsWithKeyword(clause, "VALUES"))
                                            existingValuesClauseFound = true;

                                    bool existingExecClauseFound = false;
                                    foreach (XmlElement clause in sqlTree.CurrentContainer.ParentNode.SelectNodes(SqlXmlConstants.ENAME_SQL_CLAUSE))
                                        if (ContentStartsWithKeyword(clause, "EXEC")
                                            || ContentStartsWithKeyword(clause, "EXECUTE"))
                                            existingExecClauseFound = true;

                                    if (!existingSelectClauseFound
                                        && !existingValuesClauseFound
                                        && !existingExecClauseFound
                                        )
                                        selectShouldntTryToStartNewStatement = true;
                                }

                                XmlElement firstEntryOfThisClause = sqlTree.GetFirstNonWhitespaceNonCommentChildElement(sqlTree.CurrentContainer);
                                if (firstEntryOfThisClause != null && firstEntryOfThisClause.Name.Equals(SqlXmlConstants.ENAME_SET_OPERATOR_CLAUSE))
                                    selectShouldntTryToStartNewStatement = true;
                            }

                            if (!selectShouldntTryToStartNewStatement)
                                sqlTree.ConsiderStartingNewStatement();

                            sqlTree.ConsiderStartingNewClause();

                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("UPDATE "))
                        {
                            if (sqlTree.NewStatementDue)
                                sqlTree.ConsiderStartingNewStatement();

                            if (!(sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                    && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CURSOR_FOR_OPTIONS)
                                    )
                                )
                            {
                                sqlTree.ConsiderStartingNewStatement();
                                sqlTree.ConsiderStartingNewClause();
                            }

                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("TO "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_PERMISSIONS_TARGET)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_PERMISSIONS_RECIPIENT, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                //I don't currently know whether there is any other place where "TO" can be used in T-SQL...
                                // TODO: look into that.
                                // -> for now, we'll just save as a random keyword without raising an error.
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (significantTokensString.StartsWith("FROM "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_PERMISSIONS_TARGET)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_PERMISSIONS_RECIPIENT, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else
                            {
                                sqlTree.ConsiderStartingNewClause();
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                                sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SELECTIONTARGET, "");
                            }
                        }
                        else if (significantTokensString.StartsWith("CASCADE ")
                            && sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                            && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_PERMISSIONS_RECIPIENT)
                            )
                        {
                            sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT, "", sqlTree.SaveNewElement(SqlXmlConstants.ENAME_DDL_WITH_CLAUSE, ""));
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("SET "))
                        {
                            XmlElement firstNonCommentSibling2 = sqlTree.GetFirstNonWhitespaceNonCommentChildElement(sqlTree.CurrentContainer);
                            if (!(
                                    firstNonCommentSibling2 != null
                                    && firstNonCommentSibling2.Name.Equals(SqlXmlConstants.ENAME_OTHERKEYWORD)
                                    && firstNonCommentSibling2.InnerText.ToUpperInvariant().StartsWith("UPDATE")
                                    )
                                )
                                sqlTree.ConsiderStartingNewStatement();

                            sqlTree.ConsiderStartingNewClause();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                        }
                        else if (significantTokensString.StartsWith("BETWEEN "))
                        {
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BETWEEN_CONDITION, "");
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_OPEN, ""));
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BETWEEN_LOWERBOUND, "");
                        }
                        else if (significantTokensString.StartsWith("AND "))
                        {
                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_BETWEEN_LOWERBOUND))
                            {
                                sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_BETWEEN_CONDITION);
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_CLOSE, ""));
                                sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_BETWEEN_UPPERBOUND, "");
                            }
                            else
                            {
                                sqlTree.EscapeAnyBetweenConditions();
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_AND_OPERATOR, ""));
                            }
                        }
                        else if (significantTokensString.StartsWith("OR "))
                        {
                            sqlTree.EscapeAnyBetweenConditions();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OR_OPERATOR, ""));
                        }
                        else if (significantTokensString.StartsWith("WITH "))
                        {
                            if (sqlTree.NewStatementDue)
                                sqlTree.ConsiderStartingNewStatement();

                            if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                                && !sqlTree.HasNonWhiteSpaceNonCommentContent(sqlTree.CurrentContainer)
                                )
                            {
                                sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CTE_WITH_CLAUSE, "");
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value, sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CONTAINER_OPEN, ""));
                                sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CTE_ALIAS, "");
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                                && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_PERMISSIONS_RECIPIENT)
                                )
                            {
                                sqlTree.MoveToAncestorContainer(2, SqlXmlConstants.ENAME_PERMISSIONS_BLOCK);
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_DDL_WITH_CLAUSE, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_PROCEDURAL_BLOCK)
                                || sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_DDL_OTHER_BLOCK)
                                )
                            {
                                sqlTree.StartNewContainer(SqlXmlConstants.ENAME_DDL_WITH_CLAUSE, token.Value, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT);
                            }
                            else if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SELECTIONTARGET))
                            {
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                            else
                            {
                                sqlTree.ConsiderStartingNewClause();
                                sqlTree.SaveNewElement(SqlXmlConstants.ENAME_OTHERKEYWORD, token.Value);
                            }
                        }
                        else if (tokenList.Count > tokenID + 1
                            && tokenList[tokenID + 1].Type == SqlTokenType.Colon
                            && !(tokenList.Count > tokenID + 2
                                && tokenList[tokenID + 2].Type == SqlTokenType.Colon
                                )
                            )
                        {
                            sqlTree.ConsiderStartingNewStatement();
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_LABEL, token.Value + tokenList[tokenID + 1].Value);
                            tokenID++;
                        }
                        else
                        {
                            //miscellaneous single-word tokens, which may or may not be statement starters and/or clause starters

                            //check for statements starting...
                            if (IsStatementStarter(token) || sqlTree.NewStatementDue)
                            {
                                sqlTree.ConsiderStartingNewStatement();
                            }

                            //check for statements starting...
                            if (IsClauseStarter(token))
                            {
                                sqlTree.ConsiderStartingNewClause();
                            }

                            string newNodeName = SqlXmlConstants.ENAME_OTHERNODE;
                            KeywordType matchedKeywordType;
                            if (KeywordList.TryGetValue(token.Value, out matchedKeywordType))
                            {
                                switch (matchedKeywordType)
                                {
                                    case KeywordType.OperatorKeyword:
                                        newNodeName = SqlXmlConstants.ENAME_ALPHAOPERATOR;
                                        break;
                                    case KeywordType.FunctionKeyword:
                                        newNodeName = SqlXmlConstants.ENAME_FUNCTION_KEYWORD;
                                        break;
                                    case KeywordType.DataTypeKeyword:
                                        newNodeName = SqlXmlConstants.ENAME_DATATYPE_KEYWORD;
                                        break;
                                    case KeywordType.OtherKeyword:
                                        sqlTree.EscapeAnySelectionTarget();
                                        newNodeName = SqlXmlConstants.ENAME_OTHERKEYWORD;
                                        break;
                                    default:
                                        throw new Exception("Unrecognized Keyword Type!");
                                }
                            }

                            sqlTree.SaveNewElement(newNodeName, token.Value);
                        }
                        break;

                    case SqlTokenType.Semicolon:
                        sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SEMICOLON, token.Value);
                        sqlTree.NewStatementDue = true;
                        break;

                    case SqlTokenType.Colon:
                        if (tokenList.Count > tokenID + 1
                            && tokenList[tokenID + 1].Type == SqlTokenType.Colon
                            )
                        {
                            sqlTree.SaveNewElement(SqlXmlConstants.ENAME_SCOPERESOLUTIONOPERATOR, token.Value + tokenList[tokenID + 1].Value);
                            tokenID++;
                        }
                        else
                        {
                            sqlTree.SaveNewElementWithError(SqlXmlConstants.ENAME_OTHEROPERATOR, token.Value);
                        }
                        break;

                    case SqlTokenType.Comma:
                        bool isCTESplitter = (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_CONTAINER_GENERALCONTENT)
                            && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_CTE_WITH_CLAUSE)
                            );

                        sqlTree.SaveNewElement(GetEquivalentSqlNodeName(token.Type), token.Value);

                        if (isCTESplitter)
                        {
                            sqlTree.MoveToAncestorContainer(1, SqlXmlConstants.ENAME_CTE_WITH_CLAUSE);
                            sqlTree.CurrentContainer = sqlTree.SaveNewElement(SqlXmlConstants.ENAME_CTE_ALIAS, "");
                        }
                        break;

                    case SqlTokenType.MultiLineComment:
                    case SqlTokenType.SingleLineComment:
                    case SqlTokenType.WhiteSpace:
                        //create in statement rather than clause if there are no siblings yet
                        if (sqlTree.PathNameMatches(0, SqlXmlConstants.ENAME_SQL_CLAUSE)
                            && sqlTree.PathNameMatches(1, SqlXmlConstants.ENAME_SQL_STATEMENT)
                            && sqlTree.CurrentContainer.SelectSingleNode("*") == null
                            )
                            sqlTree.SaveNewElementAsPriorSibling(GetEquivalentSqlNodeName(token.Type), token.Value, sqlTree.CurrentContainer);
                        else
                            sqlTree.SaveNewElement(GetEquivalentSqlNodeName(token.Type), token.Value);
                        break;

                    case SqlTokenType.BracketQuotedName:
                    case SqlTokenType.Asterisk:
                    case SqlTokenType.Period:
                    case SqlTokenType.NationalString:
                    case SqlTokenType.String:
                    case SqlTokenType.QuotedString:
                    case SqlTokenType.OtherOperator:
                    case SqlTokenType.Number:
                    case SqlTokenType.BinaryValue:
                    case SqlTokenType.MonetaryValue:
                    case SqlTokenType.PseudoName:
                        sqlTree.SaveNewElement(GetEquivalentSqlNodeName(token.Type), token.Value);
                        break;
                    default:
                        throw new Exception("Unrecognized element encountered!");
                }

                tokenID++;
            }

            if (!sqlTree.FindValidBatchEnd())
                sqlTree.ErrorFound = true;

            return sqlTree;
        }
Esempio n. 37
0
        public override XmlDocument ParseSQL(ITokenList tokenList)
        {
            XmlDocument sqlTree = base.ParseSQL(tokenList);

            // post-process is more extensible although it's slow...
            if (sqlTree != null)
            {
                List <XmlNode> uselessNodes = new List <XmlNode>();

                // workaround for "*=" and "=*" support
                foreach (XmlNode n in sqlTree.SelectNodes(XPATH_ASTERISK))
                {
                    XmlNode x = n.NextSibling;
                    if (x != null && x.NodeType == XmlNodeType.Element &&
                        (x as XmlElement).Name == SqlXmlConstants.ENAME_EQUALSSIGN && x.InnerText == SIGN_EQUAL)
                    {
                        uselessNodes.Add(n);
                        x.InnerText = SIGN_LEFT_JOIN;
                        continue;
                    }

                    x = n.PreviousSibling;
                    if (x != null && x.NodeType == XmlNodeType.Element &&
                        (x as XmlElement).Name == SqlXmlConstants.ENAME_EQUALSSIGN && x.InnerText == SIGN_EQUAL)
                    {
                        uselessNodes.Add(n);
                        x.InnerText = SIGN_RIGHT_JOIN;
                    }
                }

                RemoveUselessNodes(uselessNodes);

                // workaround for incorrect data type recognition
                foreach (XmlNode n in sqlTree.SelectNodes(XPATH_DATATYPE))
                {
                    XmlElement xe = sqlTree.CreateElement(SqlXmlConstants.ENAME_OTHERNODE);
                    foreach (XmlNode xn in n.ChildNodes)
                    {
                        xe.AppendChild(xn);
                    }
                    n.ParentNode.ReplaceChild(xe, n);
                }

                // add heading comments
                XmlElement root       = sqlTree.DocumentElement;
                XmlElement sqlClause  = root.FirstChild as XmlElement;
                XmlElement firstChild = null;

                if (sqlClause != null && sqlClause.Name == SqlXmlConstants.ENAME_SQL_STATEMENT)
                {
                    foreach (XmlNode node in sqlClause.ChildNodes)
                    {
                        XmlElement el = node as XmlElement;
                        if (el == null || el.Name == SqlXmlConstants.ENAME_WHITESPACE)
                        {
                            uselessNodes.Add(node);
                        }
                        else if (el.Name == SqlXmlConstants.ENAME_COMMENT_SINGLELINE)
                        {
                            firstChild = el;
                            break;
                        }
                        else
                        {
                            break;
                        }
                    }

                    RemoveUselessNodes(uselessNodes);
                }

                if (firstChild != null)
                {
                    string comment = firstChild.InnerText.Trim();
                    // [/Formatter] Formatted with Sybase T-SQL Formatter(ver: 1.4.3.16493) at 10-21-2013 15:47:18 +08:00 [Formatter/]
                    if (firstChild.Name == SqlXmlConstants.ENAME_COMMENT_SINGLELINE &&
                        comment.StartsWith(HEADER_BEGIN) &&
                        comment.EndsWith(HEADER_END))
                    {
                        //TODO: maybe parse the content and inform user something like "You're using an old version of SQL Formatter..."
                    }
                    else
                    {
                        firstChild = sqlTree.CreateElement(SqlXmlConstants.ENAME_COMMENT_SINGLELINE);
                        sqlClause.InsertBefore(firstChild, sqlClause.FirstChild);
                    }
                }
                else
                {
                    firstChild = sqlTree.CreateElement(SqlXmlConstants.ENAME_COMMENT_SINGLELINE);
                    if (sqlClause.HasChildNodes)
                    {
                        sqlClause.InsertBefore(firstChild, sqlClause.FirstChild);
                    }
                    else
                    {
                        sqlClause.AppendChild(firstChild);
                    }
                }

                DateTime currentDateTime = DateTime.Now;
                firstChild.InnerText = string.Format(SQL_HEADER,
                                                     System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(),
                                                     currentDateTime.ToString(DATE_FORMAT), TimeZone.CurrentTimeZone.GetUtcOffset(currentDateTime));

                // now remove extra lines before and after union/except/intersect keywords
            }

            return(sqlTree);
        }
Esempio n. 38
0
        private static void CompleteToken(ref SqlTokenizationType?currentTokenizationType, ITokenList tokenContainer, StringBuilder currentValue)
        {
            if (currentTokenizationType == null)
            {
                throw new Exception("Cannot complete Token, as there is no current Tokenization Type");
            }

            switch (currentTokenizationType)
            {
            case SqlTokenizationType.BlockComment:
                tokenContainer.Add(new Token(SqlTokenType.MultiLineComment, currentValue.ToString()));
                break;

            case SqlTokenizationType.OtherNode:
                tokenContainer.Add(new Token(SqlTokenType.OtherNode, currentValue.ToString()));
                break;

            case SqlTokenizationType.PseudoName:
                tokenContainer.Add(new Token(SqlTokenType.PseudoName, currentValue.ToString()));
                break;

            case SqlTokenizationType.SingleLineComment:
                tokenContainer.Add(new Token(SqlTokenType.SingleLineComment, currentValue.ToString()));
                break;

            case SqlTokenizationType.SingleHyphen:
                tokenContainer.Add(new Token(SqlTokenType.OtherOperator, "-"));
                break;

            case SqlTokenizationType.SingleDollar:
                tokenContainer.Add(new Token(SqlTokenType.MonetaryValue, "$"));
                break;

            case SqlTokenizationType.SingleSlash:
                tokenContainer.Add(new Token(SqlTokenType.OtherOperator, "/"));
                break;

            case SqlTokenizationType.WhiteSpace:
                tokenContainer.Add(new Token(SqlTokenType.WhiteSpace, currentValue.ToString()));
                break;

            case SqlTokenizationType.SingleN:
                tokenContainer.Add(new Token(SqlTokenType.OtherNode, "N"));
                break;

            case SqlTokenizationType.SingleExclamation:
                tokenContainer.Add(new Token(SqlTokenType.OtherNode, "!"));
                break;

            case SqlTokenizationType.SinglePipe:
                tokenContainer.Add(new Token(SqlTokenType.OtherNode, "|"));
                break;

            case SqlTokenizationType.NString:
                tokenContainer.Add(new Token(SqlTokenType.NationalString, currentValue.ToString()));
                break;

            case SqlTokenizationType.String:
                tokenContainer.Add(new Token(SqlTokenType.String, currentValue.ToString()));
                break;

            case SqlTokenizationType.QuotedString:
                tokenContainer.Add(new Token(SqlTokenType.QuotedString, currentValue.ToString()));
                break;

            case SqlTokenizationType.BracketQuotedName:
                tokenContainer.Add(new Token(SqlTokenType.BracketQuotedName, currentValue.ToString()));
                break;

            case SqlTokenizationType.OtherOperator:
            case SqlTokenizationType.SingleOtherCompoundableOperator:
                tokenContainer.Add(new Token(SqlTokenType.OtherOperator, currentValue.ToString()));
                break;

            case SqlTokenizationType.SingleZero:
                tokenContainer.Add(new Token(SqlTokenType.Number, "0"));
                break;

            case SqlTokenizationType.SinglePeriod:
                tokenContainer.Add(new Token(SqlTokenType.Period, "."));
                break;

            case SqlTokenizationType.SingleAsterisk:
                tokenContainer.Add(new Token(SqlTokenType.Asterisk, currentValue.ToString()));
                break;

            case SqlTokenizationType.Number:
            case SqlTokenizationType.DecimalValue:
            case SqlTokenizationType.FloatValue:
                tokenContainer.Add(new Token(SqlTokenType.Number, currentValue.ToString()));
                break;

            case SqlTokenizationType.BinaryValue:
                tokenContainer.Add(new Token(SqlTokenType.BinaryValue, currentValue.ToString()));
                break;

            case SqlTokenizationType.MonetaryValue:
                tokenContainer.Add(new Token(SqlTokenType.MonetaryValue, currentValue.ToString()));
                break;


            default:
                throw new Exception("Unrecognized SQL Node Type");
            }

            currentTokenizationType = null;
        }
 private string ExtractTokensString(ITokenList tokenList, IList<int> significantTokenPositions)
 {
     StringBuilder keywordSB = new StringBuilder();
     foreach (int tokenPos in significantTokenPositions)
     {
         //grr, this could be more elegant.
         if (tokenList[tokenPos].Type == SqlTokenType.Comma)
             keywordSB.Append(",");
         else
             keywordSB.Append(tokenList[tokenPos].Value.ToUpperInvariant());
         keywordSB.Append(" ");
     }
     return keywordSB.ToString();
 }