private static IEnumerable <GrammarAST> GetChildrenForDupTree(GrammarAST t) { bool isAlt = t.Type == ANTLRParser.ALT; int count = 0; for (int i = 0; i < t.ChildCount; i++) { GrammarAST child = (GrammarAST)t.GetChild(i); int ttype = child.Type; if (ttype == ANTLRParser.REWRITES || ttype == ANTLRParser.ACTION) { continue; } else if (ttype == ANTLRParser.BANG || ttype == ANTLRParser.ROOT) { foreach (GrammarAST subchild in GetChildrenForDupTree(child)) { count++; yield return(subchild); } } else { if (isAlt && child.Type == ANTLRParser.EOA && count == 0) { yield return(new GrammarAST(ANTLRParser.EPSILON, "epsilon")); } count++; yield return(child); } } }
protected virtual string GetDFALocations(HashSet <DFA> dfas) { HashSet <int> decisions = new HashSet <int>(); StringBuilder buf = new StringBuilder(); foreach (DFA dfa in dfas) { // if we aborted a DFA and redid with k=1, the backtrackin if (decisions.Contains(dfa.DecisionNumber)) { continue; } decisions.Add(dfa.DecisionNumber); buf.Append("Rule "); buf.Append(dfa.NFADecisionStartState.enclosingRule.Name); buf.Append(" decision "); buf.Append(dfa.DecisionNumber); buf.Append(" location "); GrammarAST decisionAST = dfa.NFADecisionStartState.associatedASTNode; buf.Append(decisionAST.Line); buf.Append(":"); buf.Append(decisionAST.CharPositionInLine); buf.Append(newline); } return(buf.ToString()); }
public override String ToString() { GrammarAST decisionASTNode = probe.Dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; String fileName = probe.Dfa.Nfa.Grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); st.SetAttribute("targetRules", targetRules); st.SetAttribute("alt", alt); st.SetAttribute("callSiteStates", callSiteStates); var labels = probe.GetSampleNonDeterministicInputSequence(sampleBadState); String input = probe.GetInputSequenceDisplay(labels); st.SetAttribute("input", input); return(base.ToString(st)); }
public override string ToString() { GrammarAST decisionASTNode = probe.Dfa.DecisionASTNode; line = decisionASTNode.Line; charPositionInLine = decisionASTNode.CharPositionInLine; string fileName = probe.Dfa.Nfa.Grammar.FileName; if (fileName != null) { file = fileName; } StringTemplate st = GetMessageTemplate(); string ruleName = probe.Dfa.NFADecisionStartState.enclosingRule.Name; st.SetAttribute("ruleName", ruleName); List <int> sortedAlts = new List <int>(); sortedAlts.AddRange(altsWithRecursion); sortedAlts.Sort(); st.SetAttribute("alts", sortedAlts); return(base.ToString(st)); }
public override void PrefixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { altTree = GrammarAST.DupTree(altTree); rewriteTree = GrammarAST.DupTree(rewriteTree); StripSynPred(altTree); int nextPrec = Precedence(alt); // rewrite e to be e_[rec_arg] StringTemplate refST = recRuleTemplates.GetInstanceOf("recRuleRef"); refST.SetAttribute("ruleName", ruleName); refST.SetAttribute("arg", nextPrec); altTree = ReplaceRuleRefs(altTree, refST.Render()); string altText = Text(altTree); altText = altText.Trim(); altText += "{}"; // add empty alt to prevent pred hoisting StringTemplate nameST = recRuleTemplates.GetInstanceOf("recRuleName"); nameST.SetAttribute("ruleName", ruleName); rewriteTree = ReplaceRuleRefs(rewriteTree, nameST.Render()); string rewriteText = Text(rewriteTree); prefixAlts.Add(altText + (rewriteText != null ? " " + rewriteText : "")); //System.out.println("prefixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); }
/** Build a subrule matching ^(. .*) (any tree or node). Let's use * (^(. .+) | .) to be safe. */ public StateCluster BuildWildcardTree(GrammarAST associatedAST) { StateCluster wildRoot = BuildWildcard(associatedAST); StateCluster down = BuildAtom(Label.DOWN, associatedAST); wildRoot = BuildAB(wildRoot, down); // hook in; . DOWN // make .+ StateCluster wildChildren = BuildWildcard(associatedAST); wildChildren = BuildAplus(wildChildren); wildRoot = BuildAB(wildRoot, wildChildren); // hook in; . DOWN .+ StateCluster up = BuildAtom(Label.UP, associatedAST); wildRoot = BuildAB(wildRoot, up); // hook in; . DOWN .+ UP // make optional . alt StateCluster optionalNodeAlt = BuildWildcard(associatedAST); //List alts = new List<object>(); var alts = new List <StateCluster>() { wildRoot, optionalNodeAlt }; StateCluster blk = BuildAlternativeBlock(alts); return(blk); }
protected void TrackInlineAction( GrammarAST actionAST ) { Rule r = grammar.GetRule( currentRuleName ); if ( r != null ) { r.TrackInlineAction( actionAST ); } }
public ActionAnalysisLexer(Grammar grammar, string ruleName, GrammarAST actionAST) : this(new ANTLRStringStream(actionAST.Token.Text)) { this.grammar = grammar; this.enclosingRule = grammar.GetLocallyDefinedRule(ruleName); this.actionToken = actionAST.Token; this.outerAltNum = actionAST.outerAltNum; }
/** Build what amounts to an epsilon transition with an action. * The action goes into NFA though it is ignored during analysis. * It slows things down a bit, but I must ignore predicates after * having seen an action (5-5-2008). */ public virtual StateCluster BuildAction(GrammarAST action) { NFAState left = NewState(); NFAState right = NewState(); Transition e = new Transition(new ActionLabel(action), right); left.AddTransition(e); return(new StateCluster(left, right)); }
public int CountAltsForBlock(GrammarAST t) { int n = 0; for ( int i = 0; i < t.ChildCount; i++ ) { if ( t.GetChild( i ).Type == ALT ) n++; } return n; }
public virtual void TrackRuleReferenceInAlt(GrammarAST refAST, int outerAltNum) { IList <GrammarAST> refs = altToRuleRefMap[outerAltNum].get(refAST.Text); if (refs == null) { refs = new List <GrammarAST>(); altToRuleRefMap[outerAltNum][refAST.Text] = refs; } refs.Add(refAST); }
/**Duplicate a tree, assuming this is a root node of a tree-- * duplicate that node and what's below; ignore siblings of root node. */ public static GrammarAST DupTreeNoActions(GrammarAST t, GrammarAST parent) { GrammarAST d = (GrammarAST)t.DupNode(); foreach (GrammarAST subchild in GetChildrenForDupTree(t)) { d.AddChild(DupTreeNoActions(subchild, d)); } return(d); }
public void StripSynPred(GrammarAST altAST) { GrammarAST t = (GrammarAST)altAST.GetChild(0); if (t.Type == ANTLRParser.BACKTRACK_SEMPRED || t.Type == ANTLRParser.SYNPRED || t.Type == ANTLRParser.SYN_SEMPRED) { altAST.DeleteChild(0); } }
public static bool BlockHasSynPred(GrammarAST blockAST) { GrammarAST c1 = blockAST.FindFirstType(ANTLRParser.SYN_SEMPRED); GrammarAST c2 = blockAST.FindFirstType(ANTLRParser.BACKTRACK_SEMPRED); if (c1 != null || c2 != null) return true; // System.out.println(blockAST.enclosingRuleName+ // " "+blockAST.getLine()+":"+blockAST.getColumn()+" no preds AST="+blockAST.toStringTree()); return false; }
protected override void TrackToken(GrammarAST t) { // imported token names might exist, only add if new // Might have ';'=4 in vocab import and SEMI=';'. Avoid // setting to UNASSIGNED if we have loaded ';'/SEMI if (grammar.GetTokenType(t.Text) == Label.INVALID && !tokens.ContainsKey(t.Text)) { tokens[t.Text] = Unassigned; } }
public static GrammarAST Dup(ITree t) { if (t == null) { return(null); } GrammarAST dup_t = new GrammarAST(); dup_t.Initialize(t); return(dup_t); }
/** From label A build Graph o-A->o */ public virtual StateCluster BuildAtom(int label, GrammarAST associatedAST) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; TransitionBetweenStates(left, right, label); StateCluster g = new StateCluster(left, right); return(g); }
/** Track a token ID or literal like '+' and "void" as having been referenced * somewhere within the alts (not rewrite sections) of a rule. * * This differs from Grammar.altReferencesTokenID(), which tracks all * token IDs to check for token IDs without corresponding lexer rules. */ public virtual void TrackTokenReferenceInAlt(GrammarAST refAST, int outerAltNum) { IList <GrammarAST> refs; _altToTokenRefMap[outerAltNum].TryGetValue(refAST.Text, out refs); if (refs == null) { refs = new List <GrammarAST>(); _altToTokenRefMap[outerAltNum][refAST.Text] = refs; } refs.Add(refAST); }
public virtual void Initialize(ITree ast) { GrammarAST t = ((GrammarAST)ast); this.Token = t.Token; this.TokenStartIndex = ast.TokenStartIndex; this.TokenStopIndex = ast.TokenStopIndex; this.enclosingRuleName = t.enclosingRuleName; this.setValue = t.setValue; this.blockOptions = t.blockOptions; this.outerAltNum = t.outerAltNum; }
internal virtual void Stats(Grammar g, StringBuilder buf) { int numDec = g.NumberOfDecisions; for (int decision = 1; decision <= numDec; decision++) { Grammar.Decision d = g.GetDecision(decision); if (d.dfa == null) { // unusued decisions in auto synpreds //System.err.println("no decision "+decision+" dfa for "+d.blockAST.toStringTree()); continue; } int k = d.dfa.MaxLookaheadDepth; Rule enclosingRule = d.dfa.NFADecisionStartState.enclosingRule; if (enclosingRule.isSynPred) { continue; // don't count synpred rules } buf.Append(g.name + "." + enclosingRule.Name + ":" + ""); GrammarAST decisionAST = d.dfa.NFADecisionStartState.associatedASTNode; buf.Append(decisionAST.Line); buf.Append(":"); buf.Append(decisionAST.CharPositionInLine); buf.Append(" decision " + decision + ":"); if (d.dfa.IsCyclic) { buf.Append(" cyclic"); } if (k != int.MaxValue) { buf.Append(" k=" + k); // fixed, no sempreds } if (d.dfa.HasSynPred) { buf.Append(" backtracks"); // isolated synpred not gated } if (d.dfa.HasSemPred) { buf.Append(" sempred"); // user-defined sempred } // else { // buf.append("undefined"); // FASerializer serializer = new FASerializer(g); // String result = serializer.serialize(d.dfa.startState); // System.err.println(result); // } buf.AppendLine(); } }
protected override void Alias(GrammarAST t, GrammarAST s) { string tokenID = t.Text; string literal = s.Text; string prevAliasLiteralID; _aliasesReverseIndex.TryGetValue(literal, out prevAliasLiteralID); if (prevAliasLiteralID != null) { // we've seen this literal before if (tokenID.Equals(prevAliasLiteralID)) { // duplicate but identical alias; might be tokens {A='a'} and // lexer rule A : 'a' ; Is ok, just return return; } // give error unless both are rules (ok if one is in tokens section) if (!(tokenRuleDefs.Contains(tokenID) && tokenRuleDefs.Contains(prevAliasLiteralID))) { // don't allow alias if A='a' in tokens section and B : 'a'; is rule. // Allow if both are rules. Will get DFA nondeterminism error later. ErrorManager.GrammarError(ErrorManager.MSG_TOKEN_ALIAS_CONFLICT, grammar, t.Token, tokenID + "=" + literal, prevAliasLiteralID); } return; // don't do the alias } int existingLiteralType = grammar.GetTokenType(literal); if (existingLiteralType != Label.INVALID) { // we've seen this before from a tokenVocab most likely // don't assign a new token type; use existingLiteralType. _tokens[tokenID] = existingLiteralType; } string prevAliasTokenID; _aliases.TryGetValue(tokenID, out prevAliasTokenID); if (prevAliasTokenID != null) { ErrorManager.GrammarError(ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT, grammar, t.Token, tokenID + "=" + literal, prevAliasTokenID); return; // don't do the alias } _aliases[tokenID] = literal; _aliasesReverseIndex[literal] = tokenID; }
protected virtual bool IsNextNonActionElementEOA(GrammarAST t) { while (t.Type == ANTLRParser.ACTION || t.Type == ANTLRParser.SEMPRED) { t = (GrammarAST)t.getNextSibling(); } if (t.Type == ANTLRParser.EOA) { return(true); } return(false); }
public override void OtherAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { altTree = GrammarAST.DupTree(altTree); rewriteTree = GrammarAST.DupTree(rewriteTree); StripSynPred(altTree); StripLeftRecursion(altTree); string altText = Text(altTree); string rewriteText = Text(rewriteTree); otherAlts.Add(altText + (rewriteText != null ? " " + rewriteText : "")); //System.out.println("otherAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); }
public virtual void DefineLabel(IToken label, GrammarAST elementRef, LabelType type) { Grammar.LabelElementPair pair = new Grammar.LabelElementPair(Grammar, label, elementRef); pair.type = type; LabelNameSpace[label.Text] = pair; switch (type) { case LabelType.Token: TokenLabels = TokenLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); TokenLabels[label.Text] = pair; break; case LabelType.WildcardTree: WildcardTreeLabels = WildcardTreeLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); WildcardTreeLabels[label.Text] = pair; break; case LabelType.WildcardTreeList: WildcardTreeListLabels = WildcardTreeListLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); WildcardTreeListLabels[label.Text] = pair; break; case LabelType.Rule: _ruleLabels = _ruleLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); _ruleLabels[label.Text] = pair; break; case LabelType.TokenList: TokenListLabels = TokenListLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); TokenListLabels[label.Text] = pair; break; case LabelType.RuleList: _ruleListLabels = _ruleListLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); _ruleListLabels[label.Text] = pair; break; case LabelType.Char: CharLabels = CharLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); CharLabels[label.Text] = pair; break; case LabelType.CharList: CharListLabels = CharListLabels ?? new Dictionary <string, Grammar.LabelElementPair>(); CharListLabels[label.Text] = pair; break; default: throw new ArgumentException(string.Format("Unexpected label type {0}.", type), "type"); } }
public static bool BlockHasSynPred(GrammarAST blockAST) { GrammarAST c1 = blockAST.FindFirstType(ANTLRParser.SYN_SEMPRED); GrammarAST c2 = blockAST.FindFirstType(ANTLRParser.BACKTRACK_SEMPRED); if (c1 != null || c2 != null) { return(true); } // System.out.println(blockAST.enclosingRuleName+ // " "+blockAST.getLine()+":"+blockAST.getColumn()+" no preds AST="+blockAST.toStringTree()); return(false); }
public void StripLeftRecursion(GrammarAST altAST) { GrammarAST rref = (GrammarAST)altAST.GetChild(0); if (rref.Type == ANTLRParser.RULE_REF && rref.Text.Equals(ruleName)) { // remove rule ref altAST.DeleteChild(0); // reset index so it prints properly GrammarAST newFirstChild = (GrammarAST)altAST.GetChild(0); altAST.TokenStartIndex = newFirstChild.TokenStartIndex; } }
protected override void TrackTokenRule(GrammarAST t, GrammarAST modifier, GrammarAST block) { // imported token names might exist, only add if new if (grammar.type == GrammarType.Lexer || grammar.type == GrammarType.Combined) { if (Rule.GetRuleType(t.Text) == RuleType.Parser) { return; } if (t.Text.Equals(Grammar.ArtificialTokensRuleName)) { // don't add Tokens rule return; } // track all lexer rules so we can look for token refs w/o // associated lexer rules. grammar.composite.lexerRules.Add(t.Text); int existing = grammar.GetTokenType(t.Text); if (existing == Label.INVALID) { tokens[t.Text] = Unassigned; } // look for "<TOKEN> : <literal> ;" pattern // (can have optional action last) if (block.HasSameTreeStructure(charAlias) || block.HasSameTreeStructure(stringAlias) || block.HasSameTreeStructure(charAlias2) || block.HasSameTreeStructure(stringAlias2)) { tokenRuleDefs.Add(t.Text); /* * Grammar parent = grammar.composite.getDelegator(grammar); * boolean importedByParserOrCombined = * parent!=null && * (parent.type==GrammarType.Lexer||parent.type==GrammarType.Parser); */ if (grammar.type == GrammarType.Combined || grammar.type == GrammarType.Lexer) { // only call this rule an alias if combined or lexer Alias(t, (GrammarAST)block.GetChild(0).GetChild(0)); } } } // else error }
protected virtual bool IsValidSimpleElementNode(GrammarAST t) { switch (t.Type) { case ANTLRParser.TREE_BEGIN: case ANTLRParser.TOKEN_REF: case ANTLRParser.CHAR_LITERAL: case ANTLRParser.STRING_LITERAL: case ANTLRParser.WILDCARD: return(true); default: return(false); } }
protected virtual bool IsNextNonActionElementEOA(GrammarAST t) { while (t.Type == ANTLRParser.ACTION || t.Type == ANTLRParser.SEMPRED) { t = (GrammarAST)t.Parent.GetChild(t.ChildIndex + 1); } if (t.Type == ANTLRParser.EOA) { return(true); } return(false); }
public void FindAllTypeImpl(int ttype, List <GrammarAST> nodes) { // check this node (the root) first if (this.Type == ttype) { nodes.Add(this); } // check children for (int i = 0; i < ChildCount; i++) { GrammarAST child = (GrammarAST)GetChild(i); child.FindAllTypeImpl(ttype, nodes); } }
/** Build an atom with all possible values in its label */ public virtual StateCluster BuildWildcard(GrammarAST associatedAST) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; Label label = new Label(nfa.grammar.TokenTypes); // char or tokens Transition e = new Transition(label, right); left.AddTransition(e); StateCluster g = new StateCluster(left, right); return(g); }
/** From set build single edge graph o->o-set->o. To conform to * what an alt block looks like, must have extra state on left. */ public virtual StateCluster BuildSet(IIntSet set, GrammarAST associatedAST) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; Label label = new Label(set); Transition e = new Transition(label, right); left.AddTransition(e); StateCluster g = new StateCluster(left, right); return(g); }
/** Track which rules have rewrite rules. Pass in the ALT node * for the alt so we can check for problems when output=template, * rewrite=true, and grammar type is tree parser. */ public virtual void TrackAltsWithRewrites(GrammarAST altAST, int outerAltNum) { if (grammar.type == GrammarType.TreeParser && grammar.BuildTemplate && grammar.GetOption("rewrite") != null && grammar.GetOption("rewrite").Equals("true") ) { GrammarAST firstElementAST = (GrammarAST)altAST.GetChild(0); grammar.sanity.EnsureAltIsSimpleNodeOrTree(altAST, firstElementAST, outerAltNum); } altsWithRewrites[outerAltNum] = true; }
/** Build what amounts to an epsilon transition with a semantic * predicate action. The pred is a pointer into the AST of * the SEMPRED token. */ public virtual StateCluster BuildSemanticPredicate(GrammarAST pred) { // don't count syn preds if (!pred.Text.StartsWith(Grammar.SynpredRulePrefix, StringComparison.OrdinalIgnoreCase)) { nfa.grammar.numberOfSemanticPredicates++; } NFAState left = NewState(); NFAState right = NewState(); Transition e = new Transition(new PredicateLabel(pred), right); left.AddTransition(e); StateCluster g = new StateCluster(left, right); return(g); }
public virtual void TernaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { }
public virtual void SetTokenPrec(GrammarAST t, int alt) { }
/** Convert e ? e : e -> ? e : e_[nextPrec] */ public override void TernaryAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { altTree = GrammarAST.DupTree(altTree); rewriteTree = GrammarAST.DupTree(rewriteTree); StripSynPred(altTree); StripLeftRecursion(altTree); int nextPrec = NextPrecedence(alt); StringTemplate refST = recRuleTemplates.GetInstanceOf("recRuleRef"); refST.SetAttribute("ruleName", ruleName); refST.SetAttribute("arg", nextPrec); altTree = ReplaceLastRuleRef(altTree, refST.Render()); string altText = Text(altTree); altText = altText.Trim(); altText += "{}"; // add empty alt to prevent pred hoisting StringTemplate nameST = recRuleTemplates.GetInstanceOf("recRuleName"); nameST.SetAttribute("ruleName", ruleName); rewriteTree = ReplaceRuleRefs(rewriteTree, "$" + nameST.Render()); string rewriteText = Text(rewriteTree); ternaryAlts.Add(alt, altText + (rewriteText != null ? " " + rewriteText : "")); //System.out.println("ternaryAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); }
protected override void TrackToken( GrammarAST t ) { // imported token names might exist, only add if new // Might have ';'=4 in vocab import and SEMI=';'. Avoid // setting to UNASSIGNED if we have loaded ';'/SEMI if ( grammar.GetTokenType( t.Text ) == Label.INVALID && !_tokens.ContainsKey( t.Text ) ) { _tokens[t.Text] = Unassigned; } }
public override void SetTokenPrec(GrammarAST t, int alt) { int ttype = g.GetTokenType(t.Text); //tokenToPrec.Add(ttype, alt); tokenToPrec[ttype] = alt; ASSOC assoc = ASSOC.left; if (t.terminalOptions != null) { object o; t.terminalOptions.TryGetValue("assoc", out o); string a = o as string; if (a != null) { if (a.Equals(ASSOC.right.ToString())) { assoc = ASSOC.right; } else { ErrorManager.Error(ErrorManager.MSG_ILLEGAL_OPTION_VALUE, "assoc", assoc); } } } ASSOC currentAssociativity; if (altAssociativity.TryGetValue(alt, out currentAssociativity)) { if (currentAssociativity != assoc) ErrorManager.Error(ErrorManager.MSG_ALL_OPS_NEED_SAME_ASSOC, alt); } else { altAssociativity.Add(alt, assoc); } //System.out.println("op " + alt + ": " + t.getText()+", assoc="+assoc); }
public GrammarAST ReplaceLastRuleRef(GrammarAST t, string name) { if (t == null) return null; GrammarAST last = null; foreach (GrammarAST rref in t.FindAllType(RULE_REF)) { last = rref; } if (last != null && last.Text.Equals(ruleName)) last.Text = name; return t; }
protected override void Alias( GrammarAST t, GrammarAST s ) { string tokenID = t.Text; string literal = s.Text; string prevAliasLiteralID; _aliasesReverseIndex.TryGetValue(literal, out prevAliasLiteralID); if ( prevAliasLiteralID != null ) { // we've seen this literal before if ( tokenID.Equals( prevAliasLiteralID ) ) { // duplicate but identical alias; might be tokens {A='a'} and // lexer rule A : 'a' ; Is ok, just return return; } // give error unless both are rules (ok if one is in tokens section) if ( !( tokenRuleDefs.Contains( tokenID ) && tokenRuleDefs.Contains( prevAliasLiteralID ) ) ) { // don't allow alias if A='a' in tokens section and B : 'a'; is rule. // Allow if both are rules. Will get DFA nondeterminism error later. ErrorManager.GrammarError( ErrorManager.MSG_TOKEN_ALIAS_CONFLICT, grammar, t.Token, tokenID + "=" + literal, prevAliasLiteralID ); } return; // don't do the alias } int existingLiteralType = grammar.GetTokenType( literal ); if ( existingLiteralType != Label.INVALID ) { // we've seen this before from a tokenVocab most likely // don't assign a new token type; use existingLiteralType. _tokens[tokenID] = existingLiteralType; } string prevAliasTokenID; _aliases.TryGetValue(tokenID, out prevAliasTokenID); if ( prevAliasTokenID != null ) { ErrorManager.GrammarError( ErrorManager.MSG_TOKEN_ALIAS_REASSIGNMENT, grammar, t.Token, tokenID + "=" + literal, prevAliasTokenID ); return; // don't do the alias } _aliases[tokenID] = literal; _aliasesReverseIndex[literal] = tokenID; }
public virtual void OtherAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { }
/** From label A build Graph o-A->o */ public virtual StateCluster BuildAtom( int label, GrammarAST associatedAST ) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; TransitionBetweenStates( left, right, label ); StateCluster g = new StateCluster( left, right ); return g; }
public virtual void SuffixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { }
public virtual StateCluster BuildAtom( GrammarAST atomAST ) { int tokenType = _nfa.Grammar.GetTokenType( atomAST.Text ); return BuildAtom( tokenType, atomAST ); }
/** Track string literals (could be in tokens{} section) */ protected override void TrackString( GrammarAST t ) { // if lexer, don't allow aliasing in tokens section if ( currentRuleName == null && grammar.type == GrammarType.Lexer ) { ErrorManager.GrammarError( ErrorManager.MSG_CANNOT_ALIAS_TOKENS_IN_LEXER, grammar, t.Token, t.Text ); return; } // in a plain parser grammar rule, cannot reference literals // (unless defined previously via tokenVocab option) // don't warn until we hit root grammar as may be defined there. if ( grammar.IsRoot && grammar.type == GrammarType.Parser && grammar.GetTokenType( t.Text ) == Label.INVALID ) { ErrorManager.GrammarError( ErrorManager.MSG_LITERAL_NOT_ASSOCIATED_WITH_LEXER_RULE, grammar, t.Token, t.Text ); } // Don't record literals for lexers, they are things to match not tokens if ( grammar.type == GrammarType.Lexer ) { return; } // otherwise add literal to token types if referenced from parser rule // or in the tokens{} section if ( ( currentRuleName == null || Rule.GetRuleType(currentRuleName) == RuleType.Parser) && grammar.GetTokenType( t.Text ) == Label.INVALID ) { _stringLiterals[t.Text] = UnassignedInParserRule; } }
/** From char 'c' build StateCluster o-intValue(c)->o */ public virtual StateCluster BuildCharLiteralAtom( GrammarAST charLiteralAST ) { int c = Grammar.GetCharValueFromGrammarCharLiteral( charLiteralAST.Text ); return BuildAtom( c, charLiteralAST ); }
protected override void TrackTokenRule( GrammarAST t, GrammarAST modifier, GrammarAST block ) { // imported token names might exist, only add if new if ( grammar.type == GrammarType.Lexer || grammar.type == GrammarType.Combined ) { if (Rule.GetRuleType(t.Text) == RuleType.Parser) { return; } if ( t.Text.Equals( Grammar.ArtificialTokensRuleName ) ) { // don't add Tokens rule return; } // track all lexer rules so we can look for token refs w/o // associated lexer rules. grammar.composite.LexerRules.Add( t.Text ); int existing = grammar.GetTokenType( t.Text ); if ( existing == Label.INVALID ) { _tokens[t.Text] = Unassigned; } // look for "<TOKEN> : <literal> ;" pattern // (can have optional action last) if ( block.HasSameTreeStructure( charAlias ) || block.HasSameTreeStructure( stringAlias ) || block.HasSameTreeStructure( charAlias2 ) || block.HasSameTreeStructure( stringAlias2 ) ) { tokenRuleDefs.Add( t.Text ); /* Grammar parent = grammar.composite.getDelegator(grammar); boolean importedByParserOrCombined = parent!=null && (parent.type==GrammarType.Lexer||parent.type==GrammarType.Parser); */ if ( grammar.type == GrammarType.Combined || grammar.type == GrammarType.Lexer ) { // only call this rule an alias if combined or lexer Alias( t, (GrammarAST)block.GetChild( 0 ).GetChild( 0 ) ); } } } // else error }
/** Build what amounts to an epsilon transition with a semantic * predicate action. The pred is a pointer into the AST of * the SEMPRED token. */ public virtual StateCluster BuildSemanticPredicate( GrammarAST pred ) { // don't count syn preds if ( !pred.Text.StartsWith( Grammar.SynpredRulePrefix, StringComparison.OrdinalIgnoreCase ) ) { _nfa.Grammar.numberOfSemanticPredicates++; } NFAState left = NewState(); NFAState right = NewState(); Transition e = new Transition( new PredicateLabel( pred ), right ); left.AddTransition( e ); StateCluster g = new StateCluster( left, right ); return g; }
/** From set build single edge graph o->o-set->o. To conform to * what an alt block looks like, must have extra state on left. */ public virtual StateCluster BuildSet( IIntSet set, GrammarAST associatedAST ) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; Label label = new Label( set ); Transition e = new Transition( label, right ); left.AddTransition( e ); StateCluster g = new StateCluster( left, right ); return g; }
public GrammarAST ReplaceRuleRefs(GrammarAST t, string name) { if (t == null) return null; foreach (GrammarAST rref in t.FindAllType(RULE_REF)) { if (rref.Text.Equals(ruleName)) rref.Text = name; } return t; }
/** For a non-lexer, just build a simple token reference atom. * For a lexer, a string is a sequence of char to match. That is, * "fog" is treated as 'f' 'o' 'g' not as a single transition in * the DFA. Machine== o-'f'->o-'o'->o-'g'->o and has n+1 states * for n characters. */ public virtual StateCluster BuildStringLiteralAtom( GrammarAST stringLiteralAST ) { if ( _nfa.Grammar.type == GrammarType.Lexer ) { StringBuilder chars = Grammar.GetUnescapedStringFromGrammarStringLiteral( stringLiteralAST.Text ); NFAState first = NewState(); NFAState last = null; NFAState prev = first; for ( int i = 0; i < chars.Length; i++ ) { int c = chars[i]; NFAState next = NewState(); TransitionBetweenStates( prev, next, c ); prev = last = next; } return new StateCluster( first, last ); } // a simple token reference in non-Lexers int tokenType = _nfa.Grammar.GetTokenType( stringLiteralAST.Text ); return BuildAtom( tokenType, stringLiteralAST ); }
/** Build an atom with all possible values in its label */ public virtual StateCluster BuildWildcard( GrammarAST associatedAST ) { NFAState left = NewState(); NFAState right = NewState(); left.associatedASTNode = associatedAST; right.associatedASTNode = associatedAST; Label label = new Label(IntervalSet.Of( _nfa.Grammar.TokenTypes )); // char or tokens Transition e = new Transition( label, right ); left.AddTransition( e ); StateCluster g = new StateCluster( left, right ); return g; }
public override void SuffixAlt(GrammarAST altTree, GrammarAST rewriteTree, int alt) { altTree = GrammarAST.DupTree(altTree); rewriteTree = GrammarAST.DupTree(rewriteTree); StripSynPred(altTree); StripLeftRecursion(altTree); StringTemplate nameST = recRuleTemplates.GetInstanceOf("recRuleName"); nameST.SetAttribute("ruleName", ruleName); rewriteTree = ReplaceRuleRefs(rewriteTree, "$" + nameST.Render()); string rewriteText = Text(rewriteTree); string altText = Text(altTree); altText = altText.Trim(); suffixAlts.Add(alt, altText + (rewriteText != null ? " " + rewriteText : "")); // System.out.println("suffixAlt " + alt + ": " + altText + ", rewrite=" + rewriteText); }
/** Build a subrule matching ^(. .*) (any tree or node). Let's use * (^(. .+) | .) to be safe. */ public StateCluster BuildWildcardTree( GrammarAST associatedAST ) { StateCluster wildRoot = BuildWildcard( associatedAST ); StateCluster down = BuildAtom( Label.DOWN, associatedAST ); wildRoot = BuildAB( wildRoot, down ); // hook in; . DOWN // make .+ StateCluster wildChildren = BuildWildcard( associatedAST ); wildChildren = BuildAplus( wildChildren ); wildRoot = BuildAB( wildRoot, wildChildren ); // hook in; . DOWN .+ StateCluster up = BuildAtom( Label.UP, associatedAST ); wildRoot = BuildAB( wildRoot, up ); // hook in; . DOWN .+ UP // make optional . alt StateCluster optionalNodeAlt = BuildWildcard( associatedAST ); //List alts = new List<object>(); var alts = new List<StateCluster>() { wildRoot, optionalNodeAlt }; StateCluster blk = BuildAlternativeBlock( alts ); return blk; }
public string Text(GrammarAST t) { if (t == null) return null; try { ITreeNodeStream input = new CommonTreeNodeStream(new ANTLRParser.grammar_Adaptor(null), t); ANTLRTreePrinter printer = new ANTLRTreePrinter(input); return printer.toString(grammar, true); } catch (Exception e) { if (e.IsCritical()) throw; ErrorManager.Error(ErrorManager.MSG_BAD_AST_STRUCTURE, e); return null; } }
/** Build what amounts to an epsilon transition with an action. * The action goes into NFA though it is ignored during analysis. * It slows things down a bit, but I must ignore predicates after * having seen an action (5-5-2008). */ public virtual StateCluster BuildAction( GrammarAST action ) { NFAState left = NewState(); NFAState right = NewState(); Transition e = new Transition( new ActionLabel( action ), right ); left.AddTransition( e ); return new StateCluster( left, right ); }