public void NextTokenIgnoreNoiseTest() { string input; GrammarTokenizer target; Token token; input = "!* block comment \r\n*!<nonTerminal>"; target = new GrammarTokenizer (input); token = target.NextTokenIgnoreNoise (); Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind); input = "!* block comment \r\n<nonTerminal>"; target = new GrammarTokenizer (input); token = target.NextTokenIgnoreNoise (); Assert.AreEqual (SymbolKind.End, token.Symbol.Kind); }
public void EmptyInputTest() { string input; GrammarTokenizer target; Token token; input = String.Empty; target = new GrammarTokenizer (input); token = target.NextToken (); Assert.AreEqual (SymbolKind.End, token.Symbol.Kind); input = null; target = new GrammarTokenizer (input); token = target.NextToken (); Assert.AreEqual (SymbolKind.End, token.Symbol.Kind); }
public void NextTokenTest() { string input; GrammarTokenizer target; Token token; input = "\0"; target = new GrammarTokenizer (input); token = target.NextToken (); Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind); token = target.NextToken (); Assert.AreEqual (SymbolKind.End, token.Symbol.Kind); input = "%{set}"; target = new GrammarTokenizer (input); token = target.NextToken (); Assert.AreEqual (SymbolKind.Error, token.Symbol.Kind); token = target.NextToken (); Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind); }
Context getContext(int index) { Context context = Context.Unknown; if (!isInComment (index)) { string lineStr = getLineUpTo (index); //tokenize the line, but don't include the trigger character var tokenizer = new GrammarTokenizer (lineStr.Remove (lineStr.Length - 1)); var firstToken = tokenizer.NextTokenIgnoreNoise (); var firstSymbolType = TokenUtil.GetDefinitionType (firstToken); if (TokenUtil.IsUserDefinitionToken (firstToken)) { Token secondToken = tokenizer.NextTokenIgnoreNoise (); if (secondToken.Symbol.Name == "=") { if (firstSymbolType==DefinitionType.Terminal) context = Context.TerminalDeclaration; else if (firstSymbolType==DefinitionType.SetName) context = Context.SetDeclaration; } else if (secondToken.Symbol.Name == "::=" && firstSymbolType==DefinitionType.NonTerminal) { context = Context.RuleDeclaration; } } else if (firstToken.Symbol.Name == "|") { context = Context.RuleDeclaration; } else if (firstToken.Symbol.Kind == SymbolKind.End) { context= Context.NewDeclaration; } } return context; }
//checks that the given line string is inside a terminal literal static bool isTerminalLiteralStart(string lineStr) { var tokenizer = new GrammarTokenizer (lineStr); var tokenList = new List<Token> (); Token token; do { token = tokenizer.NextToken(); tokenList.Add(token); } while(token.Symbol.Kind!=SymbolKind.End); //when we are inside a terminal literal, the sequence would end with these tokens: // ERROR,TERMINAL,END const int ExpectedTokenCount = 3; int lastIndex = tokenList.Count - 1; if (tokenList.Count >= ExpectedTokenCount) { return tokenList [lastIndex - 2].Symbol.Kind == SymbolKind.Error && tokenList [lastIndex - 1].Symbol.Kind == SymbolKind.Terminal; } return false; }
void parse(string text) { var tokenizer = new GrammarTokenizer (text); var tempList = new List<Token> (); Token token=null; Token prev=null; while (!TokenUtil.IsEnd(token)) { token = tokenizer.NextToken (); tokenList.Add (token); if (!TokenUtil.IsNoise (token)) { if (!TokenUtil.IsNewLine (token)) { if (isDefinitionStart (token, prev) || TokenUtil.IsEnd (token)) { addDefinitionFrom (tempList); tempList.Clear (); } tempList.Add (token); } prev = token; } } }