public Terminal LookupTerminal(GrammarToken token, string name) { if (!this.terminals.ContainsKey(name)) { this.terminals[name] = new Terminal(token == GrammarToken.Symbol, name); } return this.terminals[name]; }
public Terminal LookupTerminal(GrammarToken token, string name) { if (!this.terminals.ContainsKey(name)) { this.terminals[name] = new Terminal(token == GrammarToken.Symbol, name); } return(this.terminals[name]); }
private static void AddCharToToken(char ch, ref string output, GrammarToken token, bool escaped = false) { if (token == null) { output += ch; return; } token.FindLowestOpenToken().AddChar(ch, escaped); }
public static bool Matches(List <GrammarTokenMatch> grammarTokenMatches, List <TokenMatch> lexicalTokenMatches, List <Token> tokens, int index, int length, string type) { List <GrammarTokenMatch> validMatches = grammarTokenMatches.Where((x) => x.type == type).ToList(); foreach (var priority in validMatches.Select((x) => x.precedence).Distinct().OrderBy((x) => x)) { foreach (var priorityMatch in validMatches.Where((x) => x.precedence == priority)) { List <string> terminals = priorityMatch.matches.Where((x) => lexicalTokenMatches.Exists((y) => x == y.tokenType)).ToList(); List <int> terminalMatches = new List <int>(); int terminal = 0; int i = index; while (terminal < terminals.Count) { if (tokens[i].tokenType == terminals[terminal]) { terminalMatches.Add(i); terminal++; } i++; if (i > length) { break; } } if (terminalMatches.Count == terminals.Count) { GrammarToken grammarToken = new GrammarToken(); List <int> terminalMatchIndexes = new List <int>(); for (int p = 0; p < priorityMatch.matches.Length; p++) { int terminalID = terminals.FindIndex((x) => x == priorityMatch.matches[p]); if (terminalID >= 0) { //grammarToken } } } } } return(false); }
public static void AugmentTokensWithOriginalPosition(Grammar g, GrammarAST tree) { if (tree == null) { return; } IList <GrammarAST> optionsSubTrees = tree.GetNodesWithType(ANTLRParser.ELEMENT_OPTIONS); for (int i = 0; i < optionsSubTrees.Count; i++) { GrammarAST t = optionsSubTrees[i]; CommonTree elWithOpt = (CommonTree)t.Parent; if (elWithOpt is GrammarASTWithOptions) { IDictionary <string, GrammarAST> options = ((GrammarASTWithOptions)elWithOpt).GetOptions(); if (options.ContainsKey(LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME)) { GrammarToken newTok = new GrammarToken(g, elWithOpt.Token); newTok.originalTokenIndex = int.Parse(options[LeftRecursiveRuleTransformer.TOKENINDEX_OPTION_NAME].Text); elWithOpt.Token = newTok; GrammarAST originalNode = g.ast.GetNodeWithTokenIndex(newTok.TokenIndex); if (originalNode != null) { // update the AST node start/stop index to match the values // of the corresponding node in the original parse tree. elWithOpt.TokenStartIndex = originalNode.TokenStartIndex; elWithOpt.TokenStopIndex = originalNode.TokenStopIndex; } else { // the original AST node could not be located by index; // make sure to assign valid values for the start/stop // index so toTokenString will not throw exceptions. elWithOpt.TokenStartIndex = newTok.TokenIndex; elWithOpt.TokenStopIndex = newTok.TokenIndex; } } } } }
public ActionToken(Unparser grammar, int start, GrammarToken parent) : base(grammar, start, parent) { Type = TagType.Action; }
private GrammarToken Advance() { this.tokenStartLine = this.scanner.CurrentLine; this.tokenStartColumn = this.scanner.CurrentColumn; return this.token = this.scanner.Next(); }
private void Advance() { token = scanner.Next(); //Console.WriteLine("Token: ({0}:{1})", token, scanner.yylval); }
public Production(GrammarToken leftHandSide, CompositeGrammarToken rightHandSide) { LeftHandSide = leftHandSide; RightHandSide = rightHandSide; }
private GrammarToken Advance() { this.tokenStartLine = this.scanner.CurrentLine; this.tokenStartColumn = this.scanner.CurrentColumn; return(this.token = this.scanner.Next()); }
private void ParseRhs(NonTerminal lhs) { Production production = new Production(lhs); int num = 0; while (this.token == GrammarToken.Symbol || this.token == GrammarToken.Literal || this.token == GrammarToken.Action || this.token == GrammarToken.Prec) { GrammarToken grammarToken = this.token; switch (grammarToken) { case GrammarToken.Symbol: if (this.grammar.terminals.ContainsKey(this.scanner.yylval)) { production.rhs.Add(this.grammar.terminals[this.scanner.yylval]); } else { production.rhs.Add(this.grammar.LookupNonTerminal(this.scanner.yylval)); } this.Advance(); num++; break; case GrammarToken.Literal: production.rhs.Add(this.grammar.LookupTerminal(this.token, this.scanner.yylval)); this.Advance(); num++; break; case GrammarToken.Action: { SemanticAction semanticAction = new SemanticAction(production, this.tokenStartLine, num, this.scanner.yylval); this.Advance(); if (this.token == GrammarToken.Divider || this.token == GrammarToken.SemiColon || this.token == GrammarToken.Prec) { production.semanticAction = semanticAction; } else { Grammar arg_1BA_0 = this.grammar; string arg_1B5_0 = "@"; int num2 = ++this.grammar.NumActions; NonTerminal nonTerminal = arg_1BA_0.LookupNonTerminal(arg_1B5_0 + num2.ToString()); Production production2 = new Production(nonTerminal); production2.semanticAction = semanticAction; this.grammar.AddProduction(production2); production.rhs.Add(nonTerminal); } num++; break; } default: if (grammarToken == GrammarToken.Prec) { this.Advance(); if (this.token == GrammarToken.Symbol) { production.prec = this.grammar.LookupTerminal(this.token, this.scanner.yylval).prec; this.Advance(); } else { this.scanner.ReportError("Expected symbol after %prec", new object[0]); } } break; } } this.grammar.AddProduction(production); Precedence.Calculate(production); }
public TagToken(Unparser grammar, int start, GrammarToken parent) : base(grammar, start, parent) { Type = TagType.Tag; }
private void Advance() { token = scanner.Next(); }
internal string ParseInner(string input) { if (string.IsNullOrEmpty(input)) { return(input); } var output = ""; var escaped = false; GrammarToken token = null; for (var i = 0; i < input.Length; i++) { var ch = input[i]; if (escaped) { AddCharToToken(ch, ref output, token, true); escaped = false; continue; } if (ch == '\\') { AddCharToToken(ch, ref output, token, true); escaped = true; continue; } switch (ch) { case '[': // Start new action token. var newToken = new ActionToken(this, i + 1, token); if (token == null) { token = newToken; } else { token.AddChild(newToken); } newToken.AddChar(ch); break; case ']': // Close highest action token. If any inner tokens are unfinished, they resolve to their raw text. var action = token?.FindLowestOpenOfType(TagType.Action); if (action == null) { // No open action. Add ] to text as normal. AddCharToToken(ch, ref output, token); break; } else { action.AddChar(ch); } action.Resolve(); break; case '#': // If lowest open node is a tag, close it. Otherwise, open a new tag. if (token == null) { token = new TagToken(this, i + 1, null); token.AddChar(ch); break; } var lowest = token.FindLowestOpenToken(); if (lowest.Type == TagType.Tag) { lowest.AddChar(ch); lowest.Resolve(); break; } var newTag = new TagToken(this, i + 1, lowest); lowest.AddChild(newTag); newTag.AddChar(ch); break; default: AddCharToToken(ch, ref output, token); break; } if (token != null && token.IsResolved) { output += token.Resolved; token = null; } } return(output); }