private int Compare(UncategorizedToken left, UncategorizedToken right)
{
if (left == null || right == null)
{
throw new InvalidOperationException("Failed to compare because the specified arguments contain null.");
}
return String.Compare(left.Value, right.Value);
}
protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
{
Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);
var result = new List<IToken>();
var value = token.Value;
var current = 0;
var startIndex = 0;
while (current < token.Value.Length)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
if (value[current] == '\'' || value[current] == '"')
{
if (startIndex < current)
{
result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
}
var endIndex = value.IndexOf(value[current], current + 1);
if (endIndex != -1)
{
result.Add(BuildUpToken(value[current], value.Substring(current + 1, endIndex - (current + 1))));
current = endIndex + 1;
startIndex = endIndex + 1;
Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
}
else
{
result.Add(BuildUpToken(value[current],value.Substring(current + 1)));
current = value.Length;
startIndex = value.Length;
Logger.Verbose("Extracted a quoted token '{0}'.", result.Last().Value);
}
}
else
{
current++;
}
}
if (startIndex < current)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
}
return result;
}
protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
{
Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);
var result = new List<IToken>();
var value = token.Value;
var current = 0;
var startIndex = 0;
while (current < value.Length)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
if (Char.IsWhiteSpace(value[current]))
{
if (startIndex < current)
{
var operand = value.Substring(startIndex, current - startIndex);
var symbol = mappings.Keys.FirstOrDefault(o => o.Equals(operand, StringComparison.OrdinalIgnoreCase));
if (symbol != null)
{
result.Add(mappings[symbol]);
Logger.Verbose("Extracted an operator token '{0}'.", symbol);
}
else
{
result.Add(new OperandToken(operand));
Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
}
}
startIndex = current + 1;
}
current++;
}
if (startIndex < current)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
result.Add(new OperandToken(value.Substring(startIndex, current - startIndex)));
Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
}
return result;
}
protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
{
Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName);
var result = new List<IToken>();
var value = token.Value;
var current = 0;
var startIndex = 0;
while (current < value.Length)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
foreach (var symbol in mappings.Keys.Where(o => o.Length <= value.Length - current).OrderByDescending(o => o.Length))
{
if (value.Substring(current, symbol.Length).Equals(symbol, StringComparison.OrdinalIgnoreCase))
{
if (startIndex < current)
{
result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
}
current += symbol.Length - 1;
startIndex = current + 1;
result.Add(mappings[symbol]);
Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value);
break;
}
}
current++;
}
if (startIndex < current)
{
Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex);
result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex)));
Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value);
}
return result;
}
internal IEnumerable<IToken> InvokeTokenize(UncategorizedToken token)
{
return base.Tokenize(token);
}
protected abstract IEnumerable<IToken> Tokenize(UncategorizedToken token);
protected override IEnumerable<IToken> Tokenize(UncategorizedToken token)
{
PassedUncategorizedTokens.Add(token);
return new List<IToken>();
}
