private IList <Token> Tokenise(string expression)
{
if (ExtensionMethods.IsNullOrWhiteSpace(expression))
{
return(null);
}
var expressionLength = expression.Length;
var operators = _registeredOperators.OrderByDescending(op => op.Key.Length);
var tokens = new List <Token>();
IList <char> unrecognised = null;
var index = 0;
while (index < expressionLength)
{
var lengthProcessed = 0;
bool foundUnrecognisedCharacter = false;
// Functions would tend to have longer tags so check for these first.
foreach (var kvp in _registeredFunctions.OrderByDescending(f => f.Key.Length))
{
var lookAhead = expression.Substring(index, Math.Min(kvp.Key.Length, expressionLength - index));
if (CheckForTag(kvp.Key, lookAhead, _options))
{
CheckForUnrecognised(unrecognised, tokens, index);
lengthProcessed = kvp.Key.Length;
tokens.Add(new Token(lookAhead, index));
break;
}
}
if (lengthProcessed == 0)
{
// Loop through and find any matching operators.
foreach (var op in operators)
{
var lookAhead = expression.Substring(index, Math.Min(op.Key.Length, expressionLength - index));
if (CheckForTag(op.Key, lookAhead, _options))
{
CheckForUnrecognised(unrecognised, tokens, index);
lengthProcessed = op.Key.Length;
tokens.Add(new Token(lookAhead, index));
break;
}
}
}
// If an operator wasn't found then process the current character.
if (lengthProcessed == 0)
{
var character = expression[index];
if (character == '[')
{
char closingCharacter = ']';
if (!CanGetString(expression, index, closingCharacter))
{
throw new MissingTokenException($"Missing closing token '{closingCharacter}'", closingCharacter);
}
var variable = expression.SubstringUpTo(index, closingCharacter);
CheckForUnrecognised(unrecognised, tokens, index);
tokens.Add(new Token(variable, index));
lengthProcessed = variable.Length;
}
else if (char.IsDigit(character))
{
var number = GetNumber(expression, index);
CheckForUnrecognised(unrecognised, tokens, index);
tokens.Add(new Token(number, index));
lengthProcessed = number.Length;
}
else if (IsQuote(character))
{
if (!CanGetString(expression, index, character))
{
throw new MissingTokenException($"Missing closing token '{character}'", character);
}
var text = GetString(expression, index, character);
CheckForUnrecognised(unrecognised, tokens, index);
tokens.Add(new Token(text, index));
lengthProcessed = text.Length;
}
else if (character == DateSeparator)
{
if (!CanGetString(expression, index, character))
{
throw new MissingTokenException($"Missing closing token '{character}'", character);
}
// Ignore the first # when checking to allow us to find the second.
var dateString = "#" + expression.SubstringUpTo(index + 1, DateSeparator);
CheckForUnrecognised(unrecognised, tokens, index);
tokens.Add(new Token(dateString, index));
lengthProcessed = dateString.Length;
}
else if (character == ParameterSeparator)
{
CheckForUnrecognised(unrecognised, tokens, index);
tokens.Add(new Token(character.ToString(), index));
lengthProcessed = 1;
}
else if ((character == 't' || character == 'T') && CanExtractValue(expression, expressionLength, index, "true"))
{
CheckForUnrecognised(unrecognised, tokens, index);
var trueString = ExtractValue(expression, expressionLength, index, "true");
if (!ExtensionMethods.IsNullOrWhiteSpace(trueString))
{
tokens.Add(new Token(trueString, index));
lengthProcessed = 4;
}
}
else if ((character == 'f' || character == 'F') && CanExtractValue(expression, expressionLength, index, "false"))
{
CheckForUnrecognised(unrecognised, tokens, index);
var falseString = ExtractValue(expression, expressionLength, index, "false");
if (!ExtensionMethods.IsNullOrWhiteSpace(falseString))
{
tokens.Add(new Token(falseString, index));
lengthProcessed = 5;
}
}
else if ((character == 'n' || character == 'N') && CanExtractValue(expression, expressionLength, index, "null")) // Check for null
{
CheckForUnrecognised(unrecognised, tokens, index);
var nullString = ExtractValue(expression, expressionLength, index, "null");
if (!ExtensionMethods.IsNullOrWhiteSpace(nullString))
{
tokens.Add(new Token(nullString, index));
lengthProcessed = 4;
}
}
else if (!char.IsWhiteSpace(character))
{
// If we don't recognise this item then we had better keep going until we find something we know about.
if (unrecognised == null)
{
unrecognised = new List <char>();
}
foundUnrecognisedCharacter = true;
unrecognised.Add(character);
}
}
// Clear down the unrecognised buffer;
if (!foundUnrecognisedCharacter)
{
CheckForUnrecognised(unrecognised, tokens, index);
unrecognised = null;
}
index += (lengthProcessed == 0) ? 1 : lengthProcessed;
}
// Double check whether the last part is unrecognised.
CheckForUnrecognised(unrecognised, tokens, index);
return(tokens);
}
private static bool CanGetString(string expression, int startIndex, char quoteCharacter)
{
return(!ExtensionMethods.IsNullOrWhiteSpace(GetString(expression, startIndex, quoteCharacter)));
}
