/// <summary>
/// Enumerates through the tokens, searching for tokens XX,XX,XX where XX are arguments to the function and possibly a sub expression.
/// When a ')' is found then the end of the arguments is presumed to have been found.
/// </summary>
/// <param name="tokensEnum">Expected to be currently pointing to the name of the function. The next token SHOULD be a '('</param>
private IConstruct[] GetFunctionArguments(PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
{
var arguments = new List <IConstruct>();
var functionName = tokensEnum.Current.Value;
if (!(tokensEnum.MoveNext() && tokensEnum.Current.Type == TokenType.LeftParenthesis))
{
throw new InvalidOperationException(String.Format("{0} arguments; first token should be '(' not '{1}'", functionName, tokensEnum.Current.Value));
}
else if (tokensEnum.Current.Type == TokenType.LeftParenthesis && tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.RightParenthesis)
{
// No arguments were specified - empty parentheses were specified
tokensEnum.MoveNext(); // consume the left parenthesis token and point it to the right parenthesis token - i.e. the end of the function
}
else
{
bool reachedEndOfArguments = false;
while (!reachedEndOfArguments)
{
arguments.Add(GetConstructFromTokens(GetFunctionArgumentTokens(functionName, tokensEnum, currentVariables), currentVariables));
// tokensEnum.Current will be the last token processed by GetFunctionArgumentTokens()
if (tokensEnum.Current.Type == TokenType.RightParenthesis)
{
reachedEndOfArguments = true;
}
}
}
return(arguments.ToArray());
}
private string[] GetCommandArguments(PeekableEnumerator <Token> tokensEnum)
{
var arguments = new List <string>();
var functionName = tokensEnum.Current.Value;
while (tokensEnum.MoveNext())
{
arguments.Add(tokensEnum.Current.Value);
}
return(arguments.ToArray());
}
private IConstruct TranslateIdentifierToken(PeekableEnumerator <Token> tokensEnum, bool isInitial)
{
var identifierToken = tokensEnum.Current;
var reservedWordForToken = reservedWords.SingleOrDefault(reserverWord => identifierToken == reserverWord.Word);
var commandForToken = Program.Commands.SingleOrDefault(aCommand => identifierToken == aCommand.Name);
if (reservedWordForToken != null)
{
return(reservedWordForToken.Construct);
}
else if (commandForToken != null && isInitial)
{
return(new CommandOperation(commandForToken, GetCommandArguments(tokensEnum)));
}
else
{
if (tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.LeftParenthesis)
{
var functionForToken = Program.Functions.SingleOrDefault(aFunction => identifierToken == aFunction.Name);
if (functionForToken == null)
{
return(new Error(String.Format("Function '{0}' is undefined", identifierToken)));
}
else
{
return(new FunctionOperation(functionForToken, GetFunctionArguments(tokensEnum)));
}
}
else
{
// ensure there is only one Variable instance for the same variable name
var variable = Program.Variables.SingleOrDefault(aVariable => identifierToken == aVariable.Name);
if (variable == null)
{
// return null;
var newVariable = new Variable(tokensEnum.Current.Value);
Program.Variables.Add(newVariable);
return(newVariable);
}
else
{
return(variable);
}
}
}
}
/// <summary>
/// Translates an identifier as either a function, variable name or key word.
/// A function will match a registered function name and have a left parenthesis token following it, otherwise it is a variable.
/// </summary>
private IConstruct TranslateIdentifierToken(PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
{
var identifierToken = tokensEnum.Current;
var reservedWordForToken = reservedWords.SingleOrDefault(reserverWord => identifierToken == reserverWord.Word);
if (reservedWordForToken != null)
{
return(reservedWordForToken.Construct);
}
else
{
if (tokensEnum.CanPeek && tokensEnum.Peek.Type == TokenType.LeftParenthesis)
{
var functionForToken = allFunctions.SingleOrDefault(aFunction => identifierToken == aFunction.Name);
if (functionForToken == null)
{
throw new InvalidOperationException(String.Format("Function '{0}' is undefined", identifierToken));
}
else
{
return(new FunctionOperation(functionForToken, GetFunctionArguments(tokensEnum, currentVariables)));
}
}
else
{
// ensure there is only one Variable instance for the same variable name
var variable = currentVariables.SingleOrDefault(aVariable => identifierToken == aVariable.Name);
if (variable == null)
{
var newVariable = new Variable(tokensEnum.Current.Value);
currentVariables.Add(newVariable);
return(newVariable);
}
else
{
return(variable);
}
}
}
}
/// <summary>
/// Gets the function's next argument's tokens by traversing the tokens until the next , or ) is found (which is not within a function).
/// Does not return the , or ) character that terminated the argument expression - it is also consumed.
/// </summary>
/// <param name="functionName">Only used in order to provide useful exceptions / errors.</param>
/// <param name="tokensEnum">Should be pointing to the token that indicates the start of a function argument; either a ( or , character.</param>
private static List <Token> GetFunctionArgumentTokens(string functionName, PeekableEnumerator <Token> tokensEnum, List <Variable> currentVariables)
{
var argumentTokens = new List <Token> ();
int functionDepth = 0;
bool reachedEndOfArgument = false;
while (!reachedEndOfArgument && tokensEnum.MoveNext())
{
var token = tokensEnum.Current;
// found the argument's terminating comma or right parenthesis
if (functionDepth == 0 && (token.Type == TokenType.Comma || token.Type == TokenType.RightParenthesis))
{
reachedEndOfArgument = true;
}
else
{
argumentTokens.Add(token);
if (token.Type == TokenType.LeftParenthesis)
{
functionDepth++;
}
else if (token.Type == TokenType.RightParenthesis)
{
functionDepth--;
}
}
}
if (argumentTokens.Count == 0)
{
throw new InvalidOperationException(String.Format("{0} has an empty argument", functionName));
}
else if (!reachedEndOfArgument)
{
throw new InvalidOperationException(String.Format("{0} is missing a terminating argument character; ',' or ')'", functionName));
}
return(argumentTokens);
}
/// <summary>
/// Preliminary tokenization of the expression.
/// Tokenizes numeric values, alpha values, parentheses, commas and other tokens.
/// Any whitespace is removed.
/// </summary>
public static List <Token> Parse(string expression)
{
const char LeftParenthesis = '(';
const char RightParenthesis = ')';
const char Comma = ',';
const char NumericNegative = '-';
const char DateTimeDelimiter = '#';
var whitespaceCharacters = new[] { ' ', '\t' };
var numericCharacters = new[] { '.', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
// Identifier can contain . to support JSON Path
var identifierCharacters = new[] { '_', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z' };
var identifierSecondaryCharacters = new[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.' /* Fred*/ }; // other characters that can be used as identifiers - but cannot be a starting character
var textDelimiters = new[] { '\"', '\'' };
bool isNumericNegative = false;
bool parsingText = false;
bool parsingDateTime = false;
var tokens = new List <Token>();
var currentTokenType = TokenType.Other;
var currentToken = String.Empty;
char currentTextDelimiter = '\0';
var characterTokenType = TokenType.Other;
var expressionEnumerator = new PeekableEnumerator <char>(expression);
var characterString = String.Empty;
while (expressionEnumerator.MoveNext())
{
var tokenIsSeparateCharacter = false;
var character = expressionEnumerator.Current;
// if the character is a '-' and the subsequent character is a numeric character then this is a negative number.
// otherwise it is some other character TokenType.Other -- probably a subtraction operator.
isNumericNegative = character == NumericNegative && expressionEnumerator.CanPeek && numericCharacters.Contains(expressionEnumerator.Peek);
if (textDelimiters.Contains(character) || parsingText)
{
if (textDelimiters.Contains(character) && !parsingText) // started parsing
{
characterTokenType = TokenType.Text;
characterString = String.Empty; // consume character
currentTextDelimiter = character;
parsingText = true;
}
else if (character == currentTextDelimiter && parsingText) // finished parsing
{
characterString = String.Empty; // consume character
parsingText = false;
}
else
{
characterString = character.ToString();
}
}
else if (character == DateTimeDelimiter || parsingDateTime)
{
if (!parsingDateTime) // started parsing
{
characterTokenType = TokenType.DateTime;
characterString = String.Empty; // consume character
parsingDateTime = true;
}
else if (character == DateTimeDelimiter) // finished parsing
{
characterString = String.Empty; // consume character
parsingDateTime = false;
}
else
{
characterString = character.ToString();
}
}
else if (whitespaceCharacters.Contains(character))
{
characterTokenType = TokenType.Whitespace;
characterString = String.Empty; // consume character
}
//else if(character == '.' && currentTokenType == TokenType.Identifier)
//{
//}
else if (identifierCharacters.Contains(character) || (currentTokenType == TokenType.Identifier && identifierSecondaryCharacters.Contains(character)))
{
characterTokenType = TokenType.Identifier;
characterString = character.ToString();
}
else if (numericCharacters.Contains(character) || isNumericNegative)
{
characterTokenType = TokenType.Number;
characterString = character.ToString();
}
else if (character == LeftParenthesis)
{
characterTokenType = TokenType.LeftParenthesis;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else if (character == RightParenthesis)
{
characterTokenType = TokenType.RightParenthesis;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else if (character == Comma)
{
characterTokenType = TokenType.Comma;
characterString = character.ToString();
tokenIsSeparateCharacter = true;
}
else
{
characterTokenType = TokenType.Other;
characterString = character.ToString();
}
if (currentTokenType == characterTokenType && !tokenIsSeparateCharacter)
{
currentToken += characterString;
}
else
{
if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
{
tokens.Add(new Token(currentToken, currentTokenType));
}
currentToken = characterString;
currentTokenType = characterTokenType;
}
}
if (currentToken.Length > 0 || currentTokenType == TokenType.Text)
{
tokens.Add(new Token(currentToken, currentTokenType));
}
return(tokens);
}
private List <TranslatedToken> TranslateTokens(List <Token> tokens)
{
var translatedTokens = new List <TranslatedToken>();
var tokensEnum = new PeekableEnumerator <Token>(tokens);
bool isInitial = true;
while (tokensEnum.MoveNext())
{
var token = tokensEnum.Current;
switch (token.Type)
{
case TokenType.Number:
translatedTokens.Add(new ConstructToken(Number.Parse(token.Value)));
break;
case TokenType.Identifier:
var operationForTokenIdentifier = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForTokenIdentifier != null)
{
translatedTokens.Add(new OperatorToken(operationForTokenIdentifier));
}
else
{
translatedTokens.Add(new ConstructToken(TranslateIdentifierToken(tokensEnum, isInitial)));
}
break;
case TokenType.LeftParenthesis:
translatedTokens.Add(new LeftParenthesisToken());
break;
case TokenType.RightParenthesis:
translatedTokens.Add(new RightParenthesisToken());
break;
case TokenType.Text:
translatedTokens.Add(new ConstructToken(new Text(token.Value)));
break;
case TokenType.DateTime:
try
{
translatedTokens.Add(new ConstructToken(new DateTime(System.DateTime.Parse(token.Value))));
}
catch
{
translatedTokens.Add(new ConstructToken(new Text(token.Value)));
}
break;
case TokenType.Other:
var operationForToken = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForToken != null)
{
translatedTokens.Add(new OperatorToken(operationForToken));
}
else
{
Output.Text(token.Value + " in an unknown operation");
}
break;
case TokenType.Comma:
break;
default:
throw new NotImplementedException();
}
isInitial = false;
}
if (translatedTokens.Count == 0)
{
Output.Text(string.Format("Token Error: {0}", tokensEnum.Current));
}
return(translatedTokens);
}
/// <summary>
/// Translates the tokens into meaningful functions, operations and values.
/// </summary>
private List <TranslatedToken> TranslateTokens(List <Token> tokens, List <Variable> currentVariables)
{
var translatedTokens = new List <TranslatedToken>();
var tokensEnum = new PeekableEnumerator <Token>(tokens);
while (tokensEnum.MoveNext())
{
var token = tokensEnum.Current;
switch (token.Type)
{
case TokenType.Number:
translatedTokens.Add(new ConstructToken(Number.Parse(token.Value)));
break;
case TokenType.Identifier:
var operationForTokenIdentifier = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForTokenIdentifier != null)
{
translatedTokens.Add(new OperatorToken(operationForTokenIdentifier));
}
else
{
translatedTokens.Add(new ConstructToken(TranslateIdentifierToken(tokensEnum, currentVariables)));
}
break;
case TokenType.LeftParenthesis:
translatedTokens.Add(new LeftParenthesisToken());
break;
case TokenType.RightParenthesis:
translatedTokens.Add(new RightParenthesisToken());
break;
case TokenType.Text:
translatedTokens.Add(new ConstructToken(new Text(token.Value)));
break;
case TokenType.DateTime:
translatedTokens.Add(new ConstructToken(new DateTime(System.DateTime.Parse(token.Value))));
break;
case TokenType.Other:
var operationForToken = allOperators
.Select(item => item.Operation)
.SingleOrDefault(item => item.Token.Equals(token.Value));
if (operationForToken != null)
{
translatedTokens.Add(new OperatorToken(operationForToken));
}
else
{
throw new InvalidOperationException(token.Value + " in an unknown operation");
}
break;
default:
throw new NotImplementedException();
}
}
return(translatedTokens);
}