public void NextTokenIgnoreNoiseTest()
{
string input;
GrammarTokenizer target;
Token token;
input = "!* block comment \r\n*!<nonTerminal>";
target = new GrammarTokenizer (input);
token = target.NextTokenIgnoreNoise ();
Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);
input = "!* block comment \r\n<nonTerminal>";
target = new GrammarTokenizer (input);
token = target.NextTokenIgnoreNoise ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
}
public void EmptyInputTest()
{
string input;
GrammarTokenizer target;
Token token;
input = String.Empty;
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
input = null;
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
}
public void NextTokenTest()
{
string input;
GrammarTokenizer target;
Token token;
input = "\0";
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
input = "%{set}";
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);
}
Context getContext(int index)
{
Context context = Context.Unknown;
if (!isInComment (index))
{
string lineStr = getLineUpTo (index);
//tokenize the line, but don't include the trigger character
var tokenizer = new GrammarTokenizer (lineStr.Remove (lineStr.Length - 1));
var firstToken = tokenizer.NextTokenIgnoreNoise ();
var firstSymbolType = TokenUtil.GetDefinitionType (firstToken);
if (TokenUtil.IsUserDefinitionToken (firstToken))
{
Token secondToken = tokenizer.NextTokenIgnoreNoise ();
if (secondToken.Symbol.Name == "=")
{
if (firstSymbolType==DefinitionType.Terminal)
context = Context.TerminalDeclaration;
else if (firstSymbolType==DefinitionType.SetName)
context = Context.SetDeclaration;
}
else if (secondToken.Symbol.Name == "::=" && firstSymbolType==DefinitionType.NonTerminal)
{
context = Context.RuleDeclaration;
}
}
else if (firstToken.Symbol.Name == "|")
{
context = Context.RuleDeclaration;
}
else if (firstToken.Symbol.Kind == SymbolKind.End)
{
context= Context.NewDeclaration;
}
}
return context;
}
//checks that the given line string is inside a terminal literal
static bool isTerminalLiteralStart(string lineStr)
{
var tokenizer = new GrammarTokenizer (lineStr);
var tokenList = new List<Token> ();
Token token;
do
{
token = tokenizer.NextToken();
tokenList.Add(token);
} while(token.Symbol.Kind!=SymbolKind.End);
//when we are inside a terminal literal, the sequence would end with these tokens:
// ERROR,TERMINAL,END
const int ExpectedTokenCount = 3;
int lastIndex = tokenList.Count - 1;
if (tokenList.Count >= ExpectedTokenCount)
{
return
tokenList [lastIndex - 2].Symbol.Kind == SymbolKind.Error &&
tokenList [lastIndex - 1].Symbol.Kind == SymbolKind.Terminal;
}
return false;
}
void parse(string text)
{
var tokenizer = new GrammarTokenizer (text);
var tempList = new List<Token> ();
Token token=null;
Token prev=null;
while (!TokenUtil.IsEnd(token))
{
token = tokenizer.NextToken ();
tokenList.Add (token);
if (!TokenUtil.IsNoise (token))
{
if (!TokenUtil.IsNewLine (token))
{
if (isDefinitionStart (token, prev) || TokenUtil.IsEnd (token))
{
addDefinitionFrom (tempList);
tempList.Clear ();
}
tempList.Add (token);
}
prev = token;
}
}
}