public void NextTokenTest()
{
string input;
GrammarTokenizer target;
Token token;
input = "\0";
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
input = "%{set}";
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind);
}
public void EmptyInputTest()
{
string input;
GrammarTokenizer target;
Token token;
input = String.Empty;
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
input = null;
target = new GrammarTokenizer (input);
token = target.NextToken ();
Assert.AreEqual (SymbolKind.End, token.Symbol.Kind);
}
//checks that the given line string is inside a terminal literal
static bool isTerminalLiteralStart(string lineStr)
{
var tokenizer = new GrammarTokenizer (lineStr);
var tokenList = new List<Token> ();
Token token;
do
{
token = tokenizer.NextToken();
tokenList.Add(token);
} while(token.Symbol.Kind!=SymbolKind.End);
//when we are inside a terminal literal, the sequence would end with these tokens:
// ERROR,TERMINAL,END
const int ExpectedTokenCount = 3;
int lastIndex = tokenList.Count - 1;
if (tokenList.Count >= ExpectedTokenCount)
{
return
tokenList [lastIndex - 2].Symbol.Kind == SymbolKind.Error &&
tokenList [lastIndex - 1].Symbol.Kind == SymbolKind.Terminal;
}
return false;
}
void parse(string text)
{
var tokenizer = new GrammarTokenizer (text);
var tempList = new List<Token> ();
Token token=null;
Token prev=null;
while (!TokenUtil.IsEnd(token))
{
token = tokenizer.NextToken ();
tokenList.Add (token);
if (!TokenUtil.IsNoise (token))
{
if (!TokenUtil.IsNewLine (token))
{
if (isDefinitionStart (token, prev) || TokenUtil.IsEnd (token))
{
addDefinitionFrom (tempList);
tempList.Clear ();
}
tempList.Add (token);
}
prev = token;
}
}
}
