public void Prepare(CompilerContext context, ISourceStream source)
{
_context = context;
_source = source;
_currentToken = null;
_bufferedTokens.Clear();
ResetSource();
}
public bool ClearAuthTokens()
{
if (TokenList == null)
{
return(false);
}
TokenList.Clear();
return(true);
}
public void TokenList_Simple()
{
TokenList tokens = new TokenList();
Assert.AreEqual(0, tokens.Count);
CssToken[] expect = new CssToken[]
{
new CssToken(CssTokenType.Asterisk, 10, 1),
new CssToken(CssTokenType.Asterisk, 20, 1),
new CssToken(CssTokenType.Asterisk, 30, 1),
new CssToken(CssTokenType.Asterisk, 40, 1),
new CssToken(CssTokenType.Asterisk, 50, 1),
new CssToken(CssTokenType.Asterisk, 60, 1),
};
// The token collection is supposed to automatically sort its contents,
// so add the tokens in a weird order.
tokens.Add(expect[3]);
tokens.Add(expect[0]);
tokens.Add(expect[1]);
tokens.Add(expect[4]);
tokens.Add(expect[2]);
tokens.Insert(tokens.Count, expect[5]);
Assert.AreEqual(expect.Length, tokens.Count);
for (int i = 0; i < expect.Length; i++)
{
Assert.AreEqual(expect[i], tokens[i]);
Assert.IsTrue(tokens.Contains(expect[i]));
Assert.AreEqual(i, tokens.IndexOf(expect[i]));
}
// Test the binary search for the token collection
Assert.AreEqual(0, tokens.FindInsertIndex(0, beforeExisting: true));
Assert.AreEqual(0, tokens.FindInsertIndex(10, beforeExisting: true));
Assert.AreEqual(1, tokens.FindInsertIndex(10, beforeExisting: false));
Assert.AreEqual(3, tokens.FindInsertIndex(35, beforeExisting: true));
Assert.AreEqual(3, tokens.FindInsertIndex(35, beforeExisting: false));
Assert.AreEqual(4, tokens.FindInsertIndex(50, beforeExisting: true));
Assert.AreEqual(5, tokens.FindInsertIndex(50, beforeExisting: false));
Assert.AreEqual(6, tokens.FindInsertIndex(61, beforeExisting: true));
Assert.AreEqual(6, tokens.FindInsertIndex(61, beforeExisting: false));
Assert.AreEqual(6, tokens.FindInsertIndex(100, beforeExisting: true));
Assert.AreEqual(6, tokens.FindInsertIndex(100, beforeExisting: false));
Assert.IsTrue(tokens.Remove(expect[2]));
Assert.AreEqual(expect.Length - 1, tokens.Count);
Assert.AreEqual(expect[3], tokens[2]);
tokens.Clear();
Assert.AreEqual(0, tokens.Count);
}
public void BeginScan(CompilerContext context)
{
_context = context;
_bufferedTokens.Clear();
//create streams
FilteredStream = UnfilteredStream = CreateUnfilteredTokenStream();
//chain all token filters
foreach (TokenFilter filter in _data.TokenFilters)
{
FilteredStream = filter.BeginFiltering(context, FilteredStream);
}
FilteredTokenEnumerator = FilteredStream.GetEnumerator();
}
public void ParseExpression(string FunctionString)
{
int element = 0;
TokenList.Clear();
Symbol sym = new Symbol();
bool InBracket = false;
bool InUnderScore = false;
while (element < FunctionString.Length)
{
char ch = FunctionString[element];
if (ch == '-' && FunctionString[element + 1] != ' ') // Negative number. Parser needs space for +-*/
{
numberBuffer.Add("-1");
}
else if (isDigit(ch))
{
if (InUnderScore) // Subscript variables such as x_1
{
InUnderScore = false;
letterBuffer.Clear(); // Contained in variable buffer. subscript
variableBuffer.Add(ch.ToString());
Token t = new Token("Variable", string.Join("", variableBuffer.ToArray()));
t.SymbolEnd = false;
TokenList.Add(t);
variableBuffer.Clear();
}
else
{
numberBuffer.Add(ch.ToString());
}
}
else if (ch == '.')
{
numberBuffer.Add(ch.ToString());
}
else if (ch == '_')
{
InUnderScore = true;
variableBuffer.Add(ch.ToString());
}
else if (isLetter(ch))
{
if (numberBuffer.Count > 0)
{
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Operator", "*"));
}
if (Variables.Exists(v => v == ch.ToString()))
{
letterBuffer.Clear(); // Contained in variable buffer. subscript
variableBuffer.Add(ch.ToString());
Token t = new Token("Variable", string.Join("", variableBuffer.ToArray()));
t.SymbolEnd = false;
TokenList.Add(t);
if (element != FunctionString.Length - 1 && !InBracket)
{
TokenList.Add(new Token("Operator", "*"));
}
variableBuffer.Clear();
}
else
{
letterBuffer.Add(ch.ToString());
variableBuffer.Add(ch.ToString());
}
}
else if (ch == '^') // Number's coming
{
TokenList.Add(new Token("Operator", ch.ToString()));
}
else if (ch == '/')
{
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Operator", ch.ToString()));
}
else if (ch == ' ') // Space denotes polynomial, everything before is complete
{
//TokenList[TokenList.Count - 1].SymbolEnd = true;
// Is this else if block necessary? GPG
}
else if (ch == '+')
{
emptyNumberBufferAsLiteral();
TokenList[TokenList.Count - 1].SymbolEnd = (InBracket) ? false : true;
TokenList.Add(new Token("Operator", " " + ch.ToString() + " "));
}
else if (ch == '-' && FunctionString[element + 1] == ' ')
{
emptyNumberBufferAsLiteral();
TokenList[TokenList.Count - 1].SymbolEnd = true;
TokenList.Add(new Token("Operator", " " + ch.ToString() + " "));
}
else if (isLeftParenthesis(ch))
{
InBracket = true;
if (letterBuffer.Count > 0)
{
TokenList.Add(new Token("Function", string.Join("", letterBuffer.ToArray())));
letterBuffer.Clear();
variableBuffer.Clear();
}
else if (numberBuffer.Count > 0)
{
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Operator", "*"));
}
TokenList.Add(new Token("Left Parenthesis", ch.ToString()));
}
else if (isRightParenthesis(ch))
{
InBracket = false;
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Right Parenthesis", ch.ToString()));
}
element++;
}
emptyNumberBufferAsLiteral();
TokenList[TokenList.Count - 1].SymbolEnd = true;
this.symbolList = new SymbolList(TokenList);
}
public void ParseExpression(string FunctionString)
{
int charcounter = 0;
TokenList.Clear();
Symbol sym = new Symbol();
bool InBracket = false;
Tokenizer tokes = new Tokenizer();
while (charcounter < FunctionString.Length)
{
char ch = FunctionString[charcounter];
if (ch == '-' && FunctionString[charcounter + 1] != ' ') //negative number. Parser needs space for +-*/
{
numberBuffer.Add(ch.ToString());
}
else if (tokes.isDigit(ch))
{
numberBuffer.Add(ch.ToString());
}
else if (ch == '.')
{
numberBuffer.Add(ch.ToString());
}
else if (tokes.isLetter(ch))
{
if (numberBuffer.Count > 0)
{
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Operator", "*"));
}
if (Variables.Exists(v => v == ch.ToString()))
{
letterBuffer.Add(ch.ToString());
emptyLetterBufferAsVariables();
if (charcounter - 1 > 0)
{
char ch2 = FunctionString[charcounter - 1];
if (ch2 == ' ' && !InBracket)//(ch != '^')
{
TokenList[TokenList.Count - 1].SymbolEnd = true;
}
}
}
else
{
letterBuffer.Add(ch.ToString());
}
}
else if (tokes.isOperator(ch))
{
emptyNumberBufferAsLiteral();
emptyLetterBufferAsVariables();
char ch2 = FunctionString[charcounter - 1];
if (ch2 == ' ' && !InBracket)//(ch != '^')
{
if (TokenList.Count - 1 >= 0)
{
TokenList[TokenList.Count - 1].SymbolEnd = true;
}
}
if (ch == '^')
{
TokenList.Add(new Token("Operator", ch.ToString()));
}
else
{
TokenList.Add(new Token("Operator", " " + ch.ToString() + " "));
}
}
else if (tokes.isLeftParenthesis(ch))
{
InBracket = true;
if (letterBuffer.Count > 0)
{
TokenList.Add(new Token("Function", string.Join("", letterBuffer.ToArray())));
letterBuffer.Clear();
}
else if (numberBuffer.Count > 0)
{
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Operator", "*"));
}
TokenList.Add(new Token("Left Parenthesis", ch.ToString()));
}
else if (tokes.isRightParenthesis(ch))
{
InBracket = false;
emptyLetterBufferAsVariables();
emptyNumberBufferAsLiteral();
TokenList.Add(new Token("Right Parenthesis", ch.ToString()));
}
else if (tokes.isComma(ch))
{
emptyNumberBufferAsLiteral();
emptyLetterBufferAsVariables();
TokenList.Add(new Token("Function Argument Separator", ch.ToString()));
}
charcounter++;
}
emptyNumberBufferAsLiteral(true);
emptyLetterBufferAsVariables(true);
symbolList = new SymbolList(this.TokenList);
}