private int Compare(UncategorizedToken left, UncategorizedToken right)
        {
            if (left == null || right == null)
            {
                throw new InvalidOperationException("Failed to compare because the specified arguments contain null.");
            }

            return String.Compare(left.Value, right.Value);
        }
        protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
        {
            Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);

            var result = new List<IToken>();

            var value = token.Value;
            var current = 0;
            var startIndex = 0;

            while (current < token.Value.Length)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);

                if (value[current] == '\'' || value[current] == '"')
                {
                    if (startIndex < current)
                    {
                        result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
                        Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
                    }

                    var endIndex = value.IndexOf(value[current], current + 1);
                    if (endIndex != -1)
                    {
                        result.Add(BuildUpToken(value[current], value.Substring(current + 1, endIndex - (current + 1))));
                        current = endIndex + 1;
                        startIndex = endIndex + 1;
                        Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
                    }
                    else
                    {
                        result.Add(BuildUpToken(value[current],value.Substring(current + 1)));
                        current = value.Length;
                        startIndex = value.Length;
                        Logger.Verbose("Extracted a quoted token '{0}'.", result.Last().Value);
                    }
                }
                else
                {
                    current++;
                }
            }

            if (startIndex < current)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
                result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
                Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
            }

            return result;
        }
        protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
        {
            Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);

            var result = new List<IToken>();

            var value = token.Value;
            var current = 0;
            var startIndex = 0;

            while (current < value.Length)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);

                if (Char.IsWhiteSpace(value[current]))
                {
                    if (startIndex < current)
                    {
                        var operand = value.Substring(startIndex, current - startIndex);
                        var symbol = mappings.Keys.FirstOrDefault(o => o.Equals(operand, StringComparison.OrdinalIgnoreCase));
                        if (symbol != null)
                        {
                            result.Add(mappings[symbol]);
                            Logger.Verbose("Extracted an operator token '{0}'.", symbol);
                        }
                        else
                        {
                            result.Add(new OperandToken(operand));
                            Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
                        }
                    }
                    startIndex = current + 1;
                }
                current++;
            }

            if (startIndex < current)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
                result.Add(new OperandToken(value.Substring(startIndex, current - startIndex)));
                Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
            }

            return result;
        }
        protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
        {
            Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);

            var result = new List<IToken>();

            var value = token.Value;
            var current = 0;
            var startIndex = 0;

            while (current < value.Length)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);

                foreach (var symbol in mappings.Keys.Where(o => o.Length <= value.Length - current).OrderByDescending(o => o.Length))
                {
                    if (value.Substring(current, symbol.Length).Equals(symbol, StringComparison.OrdinalIgnoreCase))
                    {
                        if (startIndex < current)
                        {
                            result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
                            Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
                        }
                        current += symbol.Length - 1;
                        startIndex = current + 1;
                        result.Add(mappings[symbol]);
                        Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
                        break;
                    }
                }
                current++;
            }

            if (startIndex < current)
            {
                Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
                result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
                Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
            }

            return result;
        }
 internal IEnumerable<IToken> InvokeTokenize(UncategorizedToken token)
 {
     return base.Tokenize(token);
 }
 protected abstract IEnumerable<IToken> Tokenize(UncategorizedToken token);
 protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
 {
     PassedUncategorizedTokens.Add(token);
     return new List<IToken>();
 }