public void TestBasicTokeniser()
{
IdfTokeniser tks = new IdfTokeniser();
tks.ParseTokens("one + two * three");
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
}
/// <summary>
/// Convert infix to Reverse Polish Notation
/// </summary>
/// <param name="tks">IdfTokeniser that has parsed its tokens already</param>
/// <returns></returns>
private Queue<IdfToken> ShuntingYard(IdfTokeniser tks)
{
Stack<IdfToken> stack = new Stack<IdfToken>();
Queue<IdfToken> output = new Queue<IdfToken>();
IdfToken tk = null;
//reset the tokeniser to the start of its list
tks.Reset();
//see wikipeia Shunting Yard Article for details
while (tks.PeekToken() != null)
{
tk = tks.NextToken();
switch (tk.Type)
{
case IdfTokenType.Literal:
output.Enqueue(tk);
break;
case IdfTokenType.Function:
stack.Push(tk);
break;
case IdfTokenType.ParamSeperator:
while (stack.Peek().Type != IdfTokenType.LeftParen)
{
output.Enqueue(stack.Pop());
if (stack.Count == 0)
{
throw new IdfParseException("Mismatched parens or bad param separator");
}
}
break;
case IdfTokenType.Operator:
if (stack.Count != 0)
{
while (stack.Peek().Type == IdfTokenType.Operator)
{
if ((tk.Operator.Precedence < stack.Peek().Operator.Precedence) || (tk.Operator.Associativity == Associativity.Left && tk.Operator.Precedence <= stack.Peek().Operator.Precedence))
{
output.Enqueue(stack.Pop());
}
else
{
break;
}
if (stack.Count == 0)
break;
}
}
stack.Push(tk);
break;
case IdfTokenType.LeftParen:
stack.Push(tk);
break;
case IdfTokenType.RightParen:
while (stack.Peek().Type != IdfTokenType.LeftParen)
{
output.Enqueue(stack.Pop());
if (stack.Count == 0)
{
throw new IdfParseException("Mismatched parens");
}
}
stack.Pop();
if (stack.Count != 0)
{
if (stack.Peek().Type == IdfTokenType.Function)
{
output.Enqueue(stack.Pop());
}
}
break;
}
}
while (stack.Count != 0)
{
if (stack.Peek().Type == IdfTokenType.LeftParen || stack.Peek().Type == IdfTokenType.RightParen)
{
throw new IdfParseException("Mismatched parens");
}
output.Enqueue(stack.Pop());
}
return output;
}
public void TestAdvancedTokeniser()
{
IdfTokeniser tks = new IdfTokeniser();
tks.ParseTokens("one + two * three - andsoe(p1, p2, p3) + (1 + 5-6/7)");
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Function);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.LeftParen);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.ParamSeperator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.ParamSeperator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.RightParen);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.LeftParen);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Operator);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.Literal);
Assert.AreEqual(tks.NextToken().Type, IdfTokenType.RightParen);
Assert.AreEqual(tks.PeekToken(), null);
;
}