public void NextTokenIgnoreNoiseTest() { string input; GrammarTokenizer target; Token token; input = "!* block comment \r\n*!<nonTerminal>"; target = new GrammarTokenizer (input); token = target.NextTokenIgnoreNoise (); Assert.AreEqual (SymbolKind.Terminal, token.Symbol.Kind); input = "!* block comment \r\n<nonTerminal>"; target = new GrammarTokenizer (input); token = target.NextTokenIgnoreNoise (); Assert.AreEqual (SymbolKind.End, token.Symbol.Kind); }
Context getContext(int index) { Context context = Context.Unknown; if (!isInComment (index)) { string lineStr = getLineUpTo (index); //tokenize the line, but don't include the trigger character var tokenizer = new GrammarTokenizer (lineStr.Remove (lineStr.Length - 1)); var firstToken = tokenizer.NextTokenIgnoreNoise (); var firstSymbolType = TokenUtil.GetDefinitionType (firstToken); if (TokenUtil.IsUserDefinitionToken (firstToken)) { Token secondToken = tokenizer.NextTokenIgnoreNoise (); if (secondToken.Symbol.Name == "=") { if (firstSymbolType==DefinitionType.Terminal) context = Context.TerminalDeclaration; else if (firstSymbolType==DefinitionType.SetName) context = Context.SetDeclaration; } else if (secondToken.Symbol.Name == "::=" && firstSymbolType==DefinitionType.NonTerminal) { context = Context.RuleDeclaration; } } else if (firstToken.Symbol.Name == "|") { context = Context.RuleDeclaration; } else if (firstToken.Symbol.Kind == SymbolKind.End) { context= Context.NewDeclaration; } } return context; }