public void NextTokenIgnoreNoiseTest()
{
string input;
GrammarTokenizer target;
Token token;
input = "!* block comment \r\n*!<nonTerminal>";
target = new GrammarTokenizer (input);
token = target.NextTokenIgnoreNoise ();
Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);
input = "!* block comment \r\n<nonTerminal>";
target = new GrammarTokenizer (input);
token = target.NextTokenIgnoreNoise ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
}
Context getContext(int index)
{
Context context = Context.Unknown;
if (!isInComment (index))
{
string lineStr = getLineUpTo (index);
//tokenize the line, but don't include the trigger character
var tokenizer = new GrammarTokenizer (lineStr.Remove (lineStr.Length - 1));
var firstToken = tokenizer.NextTokenIgnoreNoise ();
var firstSymbolType = TokenUtil.GetDefinitionType (firstToken);
if (TokenUtil.IsUserDefinitionToken (firstToken))
{
Token secondToken = tokenizer.NextTokenIgnoreNoise ();
if (secondToken.Symbol.Name == "=")
{
if (firstSymbolType==DefinitionType.Terminal)
context = Context.TerminalDeclaration;
else if (firstSymbolType==DefinitionType.SetName)
context = Context.SetDeclaration;
}
else if (secondToken.Symbol.Name == "::=" && firstSymbolType==DefinitionType.NonTerminal)
{
context = Context.RuleDeclaration;
}
}
else if (firstToken.Symbol.Name == "|")
{
context = Context.RuleDeclaration;
}
else if (firstToken.Symbol.Kind == SymbolKind.End)
{
context= Context.NewDeclaration;
}
}
return context;
}
