public ShiftParserAction(BnfTerm term, ParserState newState) {
if (newState == null)
throw new Exception("ParserShiftAction: newState may not be null. term: " + term.ToString());
Term = term;
NewState = newState;
}
public PrecedenceBasedParserAction(BnfTerm shiftTerm, ParserState newShiftState, Production reduceProduction)
{
_reduceAction = new ReduceParserAction(reduceProduction);
var reduceEntry = new ConditionalEntry(CheckMustReduce, _reduceAction, "(Precedence comparison)");
ConditionalEntries.Add(reduceEntry);
DefaultAction = _shiftAction = new ShiftParserAction(shiftTerm, newShiftState);
}
public AstNodeArgs(BnfTerm term, CompilerContext context, SourceSpan span, AstNodeList childNodes)
{
Context = context;
Term = term;
Span = span;
ChildNodes = childNodes;
}
private void CollectTermsRecursive(BnfTerm term) {
if (_grammarData.AllTerms.Contains(term)) return;
_grammarData.AllTerms.Add(term);
NonTerminal nt = term as NonTerminal;
if (nt == null) return;
if (string.IsNullOrEmpty(nt.Name)) {
if (nt.Rule != null && !string.IsNullOrEmpty(nt.Rule.Name))
nt.Name = nt.Rule.Name;
else
nt.Name = "Unnamed" + (_unnamedCount++);
}
if (nt.Rule == null)
_language.Errors.AddAndThrow(GrammarErrorLevel.Error, null, Resources.ErrNtRuleIsNull, nt.Name);
//check all child elements
foreach (BnfTermList elemList in nt.Rule.Data)
for (int i = 0; i < elemList.Count; i++) {
BnfTerm child = elemList[i];
if (child == null) {
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrRuleContainsNull, nt.Name, i);
continue; //for i loop
}
//Check for nested expression - convert to non-terminal
BnfExpression expr = child as BnfExpression;
if (expr != null) {
child = new NonTerminal(null, expr);
elemList[i] = child;
}
CollectTermsRecursive(child);
}//for i
}//method
public UnparsableAst(BnfTerm bnfTerm, object astValue, Member astParentMember = null)
{
this.BnfTerm = bnfTerm;
this.AstValue = astValue;
this.AstParentMember = astParentMember;
this.IsLeftSiblingNeededForDeferredCalculation = false;
}
protected virtual Type GetDefaultNodeType(BnfTerm term)
{
if (term is NumberLiteral || term is StringLiteral)
return Context.DefaultLiteralNodeType;
if (term is IdentifierTerminal)
return Context.DefaultIdentifierNodeType;
return Context.DefaultNodeType;
}
static void VerifyTerm(ParseTreeNode parseTreeNode, BnfTerm expectedTerm, params BnfTerm[] moreExpectedTerms)
{
var allExpected = new[] { expectedTerm }.Concat(moreExpectedTerms);
if (!allExpected.Where(node => parseTreeNode.Term == node).Any())
{
throw new InvalidOperationException("expected '{0}' to be a '{1}'".FormatString(parseTreeNode, allExpected.JoinString(", ")));
}
}
internal static ExpressionUnparser.BnfTermKind DebugWriteLineBnfTermKind(this ExpressionUnparser.BnfTermKind bnfTermKind, TraceSource ts, BnfTerm bnfTerm, string extraMessage = null)
{
ts.Debug(
"{0}, kind: {1}{2}",
bnfTerm,
bnfTermKind,
extraMessage != null ? extraMessage : string.Empty
);
return bnfTermKind;
}
private void Process(BnfTerm term, ParseTreeNode node)
{
if (node.Term == term)
{
Visit(node);
return;
}
foreach (var childNode in node.ChildNodes)
{
Process(term, childNode);
}
}
private NonTerminal OrderBy(BnfTerm expression)
{
var orderingExpression = NonTerminal("orderByColumn",
expression + NonTerminal("orderingDirection", Empty | "ASC" | "DESC" | "УБЫВ" | "ВОЗР"),
ToOrderingElement);
var orderColumnList = NonTerminal("orderColumnList", null);
orderColumnList.Rule = MakePlusRule(orderColumnList, ToTerm(","), orderingExpression);
return(NonTerminal("orderClauseOpt",
Empty | Transient("order", ToTerm("ORDER") | "УПОРЯДОЧИТЬ") + by + orderColumnList,
ToOrderByClause));
}
private void CollectTermsRecursive(BnfTerm term)
{
if (_grammarData.AllTerms.Contains(term))
{
return;
}
_grammarData.AllTerms.Add(term);
NonTerminal nt = term as NonTerminal;
if (nt == null)
{
return;
}
if (string.IsNullOrEmpty(nt.Name))
{
if (nt.Rule != null && !string.IsNullOrEmpty(nt.Rule.Name))
{
nt.Name = nt.Rule.Name;
}
else
{
nt.Name = "Unnamed" + (_unnamedCount++);
}
nt.SetFlag(TermFlags.NoAstNode | TermFlags.IsTransient);
}
if (nt.Rule == null)
{
_language.Errors.AddAndThrow(GrammarErrorLevel.Error, null, Resources.ErrNtRuleIsNull, nt.Name);
}
//check all child elements
foreach (BnfTermList elemList in nt.Rule.Data)
{
for (int i = 0; i < elemList.Count; i++)
{
BnfTerm child = elemList[i];
if (child == null)
{
_language.Errors.Add(GrammarErrorLevel.Error, null, Resources.ErrRuleContainsNull, nt.Name, i);
continue; //for i loop
}
//Check for nested expression - convert to non-terminal
BnfExpression expr = child as BnfExpression;
if (expr != null)
{
child = new NonTerminal(null, expr);
elemList[i] = child;
}
CollectTermsRecursive(child);
} //for i
}
} //method
/// <summary>
/// Gets the child elements <see cref="ParseTreeNode"/> with the specified <see cref="BnfTerm"/>.
/// </summary>
/// <param name="node"></param>
/// <param name="term"></param>
/// <returns></returns>
public static IEnumerable <ParseTreeNode> Nodes(this ParseTreeNode node, BnfTerm term)
{
if (node == null)
{
throw new ArgumentNullException(nameof(node));
}
if (term == null)
{
throw new ArgumentNullException(nameof(term));
}
return(node.ChildNodes.Where(i => i.Term == term));
}
public LRItemSet SelectByReducedLookahead(BnfTerm lookahead)
{
var result = new LRItemSet();
foreach (var item in this)
{
if (item.ReducedLookaheads.Contains(lookahead))
{
result.Add(item);
}
}
return(result);
}
private bool NeedParentheses(BnfTerm flaggedOperator)
{
BnfTerm leftFlaggedOperator = GetLeftFlaggedSurroundingOperator();
BnfTerm rightFlaggedOperator = GetRightFlaggedSurroundingOperator();
return(leftFlaggedOperator != null &&
((GetPrecedence(leftFlaggedOperator) > GetPrecedence(flaggedOperator) ||
GetPrecedence(leftFlaggedOperator) == GetPrecedence(flaggedOperator) && GetAssociativity(flaggedOperator) == Associativity.Left))
||
rightFlaggedOperator != null &&
((GetPrecedence(rightFlaggedOperator) > GetPrecedence(flaggedOperator) ||
GetPrecedence(rightFlaggedOperator) == GetPrecedence(flaggedOperator) && GetAssociativity(flaggedOperator) == Associativity.Right)));
}
/// <summary>
/// Gets the first child <see cref="ParseTreeNode"/> with the specfied <see cref="BnfTerm"/>.
/// </summary>
/// <param name="parseTree"></param>
/// <param name="name"></param>
/// <returns></returns>
public static ParseTreeNode Node(this ParseTree parseTree, BnfTerm term)
{
if (parseTree == null)
{
throw new ArgumentNullException(nameof(parseTree));
}
if (term == null)
{
throw new ArgumentNullException(nameof(term));
}
return(parseTree.Root.Term == term ? parseTree.Root : null);
}
public LRItemSet SelectByCurrent(BnfTerm current)
{
var result = new LRItemSet();
foreach (var item in this)
{
if (item.Core.Current == current)
{
result.Add(item);
}
}
return(result);
}
/// <summary>
/// Gets the first child <see cref="ParseTreeNode"/> with the specified <see cref="BnfTerm"/>.
/// </summary>
/// <param name="node"></param>
/// <param name="name"></param>
/// <returns></returns>
public static ParseTreeNode Node(this ParseTreeNode node, BnfTerm term)
{
if (node == null)
{
throw new ArgumentNullException(nameof(node));
}
if (term == null)
{
throw new ArgumentNullException(nameof(term));
}
return(node.ChildNodes.Find(i => i.Term == term));
}
private static string GetName(Type domainType, BnfTerm bnfTerm)
{
string name = string.Empty;
if (bnfTerm != null)
{
name += bnfTerm.Name + "_";
}
name += "copyAs_" + domainType.Name.ToLower();
return(name);
}
//Creates closure items with "spontaneously generated" lookaheads
private bool AddClosureItems(ParserState state)
{
bool result = false;
//note that we change collection while we iterate thru it, so we have to use "for i" loop
for (int i = 0; i < state.Items.Count; i++)
{
LRItem item = state.Items[i];
BnfTerm currTerm = item.Core.Current;
if (currTerm == null || !(currTerm is NonTerminal))
{
continue;
}
//1. Add normal closure items
NtData currInfo = (NtData)currTerm.ParserData;
foreach (Production prod in currInfo.Productions)
{
LR0Item core = prod.LR0Items[0]; //item at zero index is the one that starts with dot
LRItem newItem = TryFindItem(state, core);
if (newItem == null)
{
newItem = new LRItem(state, core);
state.Items.Add(newItem);
result = true;
}
#region Comments on lookaheads processing
// The general idea of generating ("spontaneously") the lookaheads is the following:
// Let's the original item be in the form
// [A -> alpha . B beta , lset]
// where <B> is a non-terminal and <lset> is a set of lookaheads,
// <beta> is some string (B's tail in our terminology)
// Then the closure item on non-teminal B is an item
// [B -> x, firsts(beta + lset)]
// (the lookahead set is expression after the comma).
// To generate lookaheads on a closure item, we simply take "firsts"
// from the tail <beta> of the NonTerminal <B>.
// Normally if tail <beta> is nullable we would add ("propagate")
// the <lset> lookaheads from <A> to <B>.
// We dont' do it right here - we simply add a propagation link.
// We propagate all lookaheads later in a separate process.
#endregion
newItem.NewLookaheads.AddRange(item.Core.TailFirsts);
if (item.Core.TailIsNullable && !item.PropagateTargets.Contains(newItem))
{
item.PropagateTargets.Add(newItem);
}
} //foreach prod
} //for i (LRItem)
return(result);
}
public ShiftTransition(ParserState fromState, BnfTerm overTerm, LRItemSet shiftItems)
{
FromState = fromState;
OverTerm = overTerm;
ShiftItems = shiftItems;
_hashCode = unchecked (fromState.GetHashCode() - overTerm.GetHashCode());
//Add self to state's set of transitions
fromState.BuilderData.ShiftTransitions.Add(this);
//Assign self to Transition field of all shift items
foreach (var shiftItem in ShiftItems)
{
shiftItem.Transition = this;
}
}//constructor
private void RegisterMember(BnfTerm bnfTerm, IList <BnfTerm> rawEncloserBnfTerms, int rawEncloserBnfTermsIndex, int bnfTermIndexAtParse, int bnfTermIndexAtRule, Member member)
{
var encloserBnfTermsAtRule = rawEncloserBnfTerms
.Where(_bnfTerm => !(_bnfTerm is GrammarHint))
.Select(_bnfTerm => _bnfTerm is Member ? ((Member)_bnfTerm).BnfTerm : _bnfTerm)
.ToList();
var encloserBnfTermsAtParse = encloserBnfTermsAtRule
.Where(encloserBnfTermAtRule => !encloserBnfTermAtRule.IsPunctuation())
.ToList();
this.referredBnfTermAtRuleToMember.Add(new ReferredBnfTermEI(rawEncloserBnfTermsIndex, bnfTerm, bnfTermIndexAtRule), member);
this.referredBnfTermAtParseToMember.Add(new ReferredBnfTermEL(encloserBnfTermsAtParse, bnfTerm, bnfTermIndexAtParse), member);
}
} //method
#endregion
#region Calculating Tail Firsts
private void CalculateTailFirsts()
{
foreach (NonTerminal nt in _grammar.NonTerminals)
{
foreach (Production prod in nt.Productions)
{
StringSet accumulatedFirsts = new StringSet();
bool allNullable = true;
//We are going backwards in LR0Items list
for (int i = prod.LR0Items.Count - 1; i >= 0; i--)
{
LR0Item item = prod.LR0Items[i];
if (i >= prod.LR0Items.Count - 2)
{
//Last and before last items have empty tails
item.TailIsNullable = true;
item.TailFirsts.Clear();
continue;
}
BnfTerm nextTerm = prod.RValues[i + 1]; //Element after-after-dot; remember we're going in reverse direction
//if (ntElem == null) continue; //it is not NonTerminal
NonTerminal nextNt = nextTerm as NonTerminal;
bool notNullable = nextTerm is Terminal || nextNt != null && !nextNt.Nullable;
if (notNullable) //next term is not nullable (a terminal or non-nullable NonTerminal)
//term is not nullable, so we clear all old firsts and add this term
{
accumulatedFirsts.Clear();
allNullable = false;
item.TailIsNullable = false;
if (nextTerm is Terminal)
{
item.TailFirsts.Add(nextTerm.Key);//term is terminal so add its key
accumulatedFirsts.Add(nextTerm.Key);
}
else if (nextNt != null) //it is NonTerminal
{
item.TailFirsts.AddRange(nextNt.Firsts); //nonterminal
accumulatedFirsts.AddRange(nextNt.Firsts);
}
continue;
}
//if we are here, then ntElem is a nullable NonTerminal. We add
accumulatedFirsts.AddRange(nextNt.Firsts);
item.TailFirsts.AddRange(accumulatedFirsts);
item.TailIsNullable = allNullable;
} //for i
} //foreach prod
} //foreach nt
} //method
// AST node type is not specified for term {0}. Either assign Term.AstConfig.NodeType, or specify default type(s) in AstBuilder.
protected virtual Type GetDefaultNodeType(BnfTerm term)
{
if (term is NumberLiteral || term is StringLiteral)
{
return(Context.DefaultLiteralNodeType);
}
else if (term is IdentifierTerminal)
{
return(Context.DefaultIdentifierNodeType);
}
else
{
return(Context.DefaultNodeType);
}
}
protected void SetState(BnfiTermConversion source)
{
this.bnfTerm = source.bnfTerm;
this.value = source.value;
this.defaultValue = source.defaultValue;
this.isOptionalValue = source.isOptionalValue;
this.Flags = source.Flags;
this.AstConfig = source.AstConfig;
this.inverseValueConverterForUnparse = source.inverseValueConverterForUnparse;
if (this.UtokenizerForUnparse != null)
{
this.UtokenizerForUnparse = source.UtokenizerForUnparse;
}
}
protected BnfiTermCopy(Type domainType, BnfTerm bnfTerm, string name)
: base(domainType, name: name ?? GetName(domainType, bnfTerm))
{
if (bnfTerm != null)
{
// "this" BnfiTermCopy is not an independent bnfTerm, just a syntax magic for BnfiTermRecord<TType> (we were called by the Copy method)
this.IsContractible = true;
this.RuleRaw = bnfTerm.ToBnfExpression() + GrammarHelper.ReduceHere();
}
else
{
// "this" BnfiTermCopy is an independent bnfTerm
this.IsContractible = false;
}
GrammarHelper.MarkTransientForced(this); // default "transient" behavior (the Rule of this BnfiTermCopyable will contain the BnfiTerm... which actually does something)
}
private object ConvertAstValueForChild(object astValue, BnfTerm childBnfTerm)
{
/*
* If UtokenizerForUnparse is not null then this method is being called only for child priority calculations, and not for
* actually getting the children for unparse. However, if we are being called for priority calculations, we shouldn't
* throw an exception because of the unimplemented inverseValueConverterForUnparse.
* */
if (IsMainChild(childBnfTerm) && UtokenizerForUnparse == null)
{
return(this.inverseValueConverterForUnparse(astValue));
}
else
{
return(astValue);
}
}
}//method
private int GetLookaheadOccurenceCount(ParserState state, BnfTerm lookahead)
{
var result = 0;
foreach (var reduceItem in state.BuilderData.ReduceItems)
{
if (reduceItem.AllLookaheads.Contains(lookahead))
{
result++;
}
}
//add 1 if it is shift term
if (state.BuilderData.ShiftTerms.Contains(lookahead))
{
result++;
}
return(result);
}
private BnfTermKind CalculateBnfTermKind(BnfTerm current)
{
BnfTermKind currentKind;
if (this.bnfTermToBnfTermKind.TryGetValue(current, out currentKind))
{
/*
* if currentKind is Undetermined here it means that we are in a recursion,
* which means that currentKind can be only Other (i.e. not an operator, nor a parenthesis, etc.)
* */
if (currentKind == BnfTermKind.Undetermined)
{
this.bnfTermToBnfTermKind[current] = currentKind = BnfTermKind.Other;
}
return(currentKind);
}
this.bnfTermToBnfTermKind.Add(current, BnfTermKind.Undetermined);
tsParentheses.Indent();
// need to "label" every bnfterm
if (current is NonTerminal)
{
currentKind = CalculateBnfTermKindForNonTerminal((NonTerminal)current);
}
else
{
currentKind = BnfTermKind.Other;
}
// e.g. operator can be non-terminal and terminal as well (or at least they can be flagged so); if it is non-terminal, then we override currentKind
OverrideBnfTermKindInCaseOfSpecificInfo(current, ref currentKind);
tsParentheses.Unindent();
currentKind.DebugWriteLineBnfTermKind(tsParentheses, current);
Debug.Assert(currentKind != BnfTermKind.Undetermined);
this.bnfTermToBnfTermKind[current] = currentKind; // not using Add, because bnfTermToBnfTermKind[current] has already been set to Undetermined or Other
return(currentKind);
}
private NonTerminal AttributesList(BnfTerm identifier)
{
var options = StringOptions.AllowsAllEscapes | StringOptions.AllowsLineBreak;
var attributeValue = new StringLiteral("attributeValue", "'", options,
(context, node) => node.AstNode = node.Token.ValueString);
attributeValue.AddStartEnd("\"", options);
var attribute = NonTerminal("attribute", null, node => new Attribute
{
Name = (CompositeIdentifier)node.ChildNodes[0].AstNode,
Value = (string)node.ChildNodes[2].AstNode
});
var attributesList = NonTerminal("attributesList", null,
node => node.ChildNodes.Select(c => (Attribute)c.AstNode).ToList());
attribute.Rule = identifier + "=" + attributeValue;
attributesList.Rule = MakeStarRule(attributesList, attribute);
return(attributesList);
}
private NonTerminal Join(BnfTerm columnSource, BnfTerm joinCondition)
{
var joinKindOpt = NonTerminal("joinKindOpt", null);
var joinItem = NonTerminal("joinItem", null, ToJoinClause);
var joinItemList = NonTerminal("joinItemList", null);
var outerJoinKind = NonTerminal("outerJoinKind", null, TermFlags.IsTransient);
var outerKeywordOpt = NonTerminal("outerKeywordOpt", null, TermFlags.NoAstNode | TermFlags.IsPunctuation);
var on = NonTerminal("on", ToTerm("ON") | "ПО", TermFlags.NoAstNode);
outerKeywordOpt.Rule = ToTerm("OUTER") | "ВНЕШНЕЕ" | Empty;
outerJoinKind.Rule = ToTerm("FULL") | "LEFT" | "RIGHT" | "ПОЛНОЕ" | "ЛЕВОЕ" | "ПРАВОЕ";
joinItem.Rule = joinKindOpt + Transient("join", ToTerm("JOIN") | "СОЕДИНЕНИЕ") + columnSource + on +
joinCondition;
joinKindOpt.Rule = Empty | Transient("inner", ToTerm("INNER") | "ВНУТРЕННЕЕ") |
(outerJoinKind + outerKeywordOpt);
joinItemList.Rule = MakeStarRule(joinItemList, null, joinItem);
return(joinItemList);
}
private void HandleOperators(BnfTerm @operator, IEnumerable <BnfTerm> childOperators)
{
BnfTerm strongestFlaggedChildOperator = childOperators.Aggregate(
(BnfTerm)null,
(strongestFlaggedChildOperatorSoFar, childOperator) => strongestFlaggedChildOperatorSoFar == null ||
GetPrecedence(_flaggedOrDerivedOperatorToMultiOperatorInfo[childOperator].StrongestFlaggedOperator) > GetPrecedence(_flaggedOrDerivedOperatorToMultiOperatorInfo[strongestFlaggedChildOperatorSoFar].StrongestFlaggedOperator)
? _flaggedOrDerivedOperatorToMultiOperatorInfo[childOperator].StrongestFlaggedOperator
: strongestFlaggedChildOperatorSoFar
);
BnfTerm weakestFlaggedChildOperator = childOperators.Aggregate(
(BnfTerm)null,
(weakestFlaggedChildOperatorSoFar, childOperator) => weakestFlaggedChildOperatorSoFar == null ||
GetPrecedence(_flaggedOrDerivedOperatorToMultiOperatorInfo[childOperator].WeakestFlaggedOperator) > GetPrecedence(_flaggedOrDerivedOperatorToMultiOperatorInfo[weakestFlaggedChildOperatorSoFar].WeakestFlaggedOperator)
? _flaggedOrDerivedOperatorToMultiOperatorInfo[childOperator].WeakestFlaggedOperator
: weakestFlaggedChildOperatorSoFar
);
_flaggedOrDerivedOperatorToMultiOperatorInfo.Add(@operator, new MultiOperatorInfo(strongestFlaggedChildOperator, weakestFlaggedChildOperator));
}
private void OverrideBnfTermKindInCaseOfSpecificInfo(BnfTerm current, ref BnfTermKind currentKind)
{
if (IsFlaggedOperator(current))
{
currentKind = BnfTermKind.Operator;
_flaggedOrDerivedOperatorToMultiOperatorInfo[current] = new MultiOperatorInfo(current, current);
}
else if (current.IsOpenBrace())
{
currentKind = BnfTermKind.LeftParenthesis;
}
else if (current.IsCloseBrace())
{
currentKind = BnfTermKind.RightParenthesis;
}
else if (current is GrammarHint)
{
currentKind = BnfTermKind.GrammarHint;
}
}
static void ProcessParseTreeNode(BnfTerm node)
{
var nt = node as NonTerminal;
if (nt == null)
return;
if (node_names.ContainsKey (nt))
return;
string name = "AS" + ToPascalCase (nt.Name) + "AstNode";
node_names.Add (nt, name);
// process descendants
foreach (var p in nt.Productions)
foreach (var c in p.RValues)
ProcessParseTreeNode (c);
foreach (var p in nt.Productions)
Console.WriteLine ("\t// {0}", p.GetType ());
Console.WriteLine ("\tpublic partial class {0} : ActionScriptAstNode", name);
Console.WriteLine ("\t{");
Console.Write ("\t\t// {0} productions: ", nt.Productions.Count);
foreach (var p in nt.Productions)
foreach (var c in p.RValues) {
Console.Write (ToPascalCase (c.Name));
Console.Write (' ');
}
Console.WriteLine ();
if ((nt.Flags & TermFlags.IsList) != 0 && nt.Productions.Count > 0 && nt.Productions [0].RValues [0] == nt) {
var cnt = nt.Productions [0].RValues.Last () as NonTerminal;
if (cnt != null)
Console.WriteLine ("\t\tpublic {0} {1} {{ get; set; }}", node_names [cnt], ToPascalCase (cnt.Name));
} else {
foreach (var p in nt.Productions)
foreach (var c in p.RValues) {
var cnt = c as NonTerminal;
if (cnt == null)
continue;
Console.WriteLine ("\t\tpublic {0} {1} {{ get; set; }}", node_names [cnt], ToPascalCase (cnt.Name));
}
}
Console.WriteLine ("\t}");
}
//Find an LR item without hints compatible with term (either shift on term or reduce with term as lookahead);
// this item without hints would become our default. We assume that other items have hints, and when conditions
// on all these hints fail, we chose this remaining item without hints.
private static ParserAction FindDefaultAction(ParserState state, BnfTerm term)
{
//First check reduce items
var reduceItems = state.BuilderData.ReduceItems.SelectByLookahead(term as Terminal);
foreach (var item in reduceItems)
{
if (item.Core.Hints.Count == 0)
{
return(ReduceParserAction.Create(item.Core.Production));
}
}
var shiftItem = state.BuilderData.ShiftItems.SelectByCurrent(term).FirstOrDefault();
if (shiftItem != null)
{
return(new ShiftParserAction(shiftItem));
}
//if everything failed, returned first reduce item
return(null);
}
}//method
private ShiftTable GetStateShifts(ParserState state)
{
ShiftTable shifts = new ShiftTable();
LR0ItemList list;
foreach (LRItem item in state.Items)
{
BnfTerm term = item.Core.Current;
if (term == null)
{
continue;
}
LR0Item shiftedItem = item.Core.Production.LR0Items[item.Core.Position + 1];
if (!shifts.TryGetValue(term.Key, out list))
{
shifts[term.Key] = list = new LR0ItemList();
}
list.Add(shiftedItem);
} //foreach
return(shifts);
} //method
/// <summary>
/// Gets the child elements <see cref="ParseTreeNode"/> with the specified <see cref="BnfTerm"/>.
/// </summary>
/// <param name="node"></param>
/// <param name="name"></param>
/// <returns></returns>
public static IEnumerable <ParseTreeNode> Nodes(this ParseTree parseTree, BnfTerm term)
{
if (parseTree == null)
{
throw new ArgumentNullException(nameof(parseTree));
}
if (parseTree.Root == null)
{
throw new ArgumentNullException(nameof(parseTree));
}
if (parseTree.Root.Term == null)
{
throw new ArgumentNullException(nameof(parseTree));
}
if (term == null)
{
throw new ArgumentNullException(nameof(term));
}
return(parseTree.Root.Term == term ? new[] { parseTree.Root } : Enumerable.Empty <ParseTreeNode>());
}
protected BnfiTermConversion(Type domainType, BnfTerm bnfTerm, ValueIntroducer <object> valueIntroducer, ValueConverter <object, object> inverseValueConverterForUnparse,
object defaultValue, bool isOptionalValue, string name, bool astForChild)
: base(domainType, name)
{
this.IsContractible = true;
this.bnfTerm = bnfTerm;
this.isOptionalValue = isOptionalValue;
this.defaultValue = defaultValue;
if (!astForChild)
{
bnfTerm.SetFlag(TermFlags.NoAstNode);
}
this.RuleRawWithMove = isOptionalValue
? GrammarHelper.PreferShiftHere() + bnfTerm | Irony.Parsing.Grammar.CurrentGrammar.Empty
: bnfTerm.ToBnfExpression();
this.AstConfig.NodeCreator =
(context, parseTreeNode) =>
{
try
{
parseTreeNode.AstNode = GrammarHelper.ValueToAstNode(valueIntroducer(context, new ParseTreeNodeWithoutAst(parseTreeNode)), context, parseTreeNode);
}
catch (AstException e)
{
context.AddMessage(AstException.ErrorLevel, parseTreeNode.Span.Location, e.Message);
}
catch (FatalAstException e)
{
context.AddMessage(FatalAstException.ErrorLevel, parseTreeNode.Span.Location, e.Message); // although it will be abandoned anyway
e.Location = parseTreeNode.Span.Location;
throw; // handle in MultiParser
}
};
this.inverseValueConverterForUnparse = inverseValueConverterForUnparse;
}
public static Automaton CreateAutomaton(Grammar g)
{
//initialise to closure of start item
HashSet <ParseState> states = new HashSet <ParseState>();
states.Add(g.Productions.Where(a => a.Head.Equals(g.Root)).Select(a => new Item(a, 0)).Closure(g));
HashSet <ParseStateTransition> transitions = new HashSet <ParseStateTransition>();
HashSet <ParseState> sToAdd = new HashSet <ParseState>();
HashSet <ParseStateTransition> tToAdd = new HashSet <ParseStateTransition>();
do
{
sToAdd.Clear();
tToAdd.Clear();
foreach (var state in states)
{
foreach (var item in state)
{
if (item.Production.Body.Length == item.Position)
{
continue;
}
BnfTerm term = item.Production.Body[item.Position];
ParseState j = state.Goto(term, g);
sToAdd.Add(j);
tToAdd.Add(new ParseStateTransition(state, term, j));
}
}
}while (states.UnionWithAddedCount(sToAdd) != 0 | transitions.UnionWithAddedCount(tToAdd) != 0);
return(new Automaton(transitions, g));
}
private void CollectTermsRecursive(BnfTerm term) {
// Do not add pseudo terminals defined as static singletons in Grammar class (Empty, Eof, etc)
// We will never see these terminals in the input stream.
// Filter them by type - their type is exactly "Terminal", not derived class.
if (term.GetType() == typeof(Terminal)) return;
if (_grammarData.AllTerms.Contains(term)) return;
_grammarData.AllTerms.Add(term);
NonTerminal nt = term as NonTerminal;
if (nt == null) return;
if (nt.Name == null) {
if (nt.Rule != null && !string.IsNullOrEmpty(nt.Rule.Name))
nt.Name = nt.Rule.Name;
else
nt.Name = "NT" + (_unnamedCount++);
}
if (nt.Rule == null) {
_language.Errors.Add("Non-terminal " + nt.Name + " has uninitialized Rule property.");
return;
}
//check all child elements
foreach (BnfTermList elemList in nt.Rule.Data)
for (int i = 0; i < elemList.Count; i++) {
BnfTerm child = elemList[i];
if (child == null) {
_language.Errors.Add("Rule for NonTerminal " + nt.Name + " contains null as an operand in position " + i.ToString() + " in one of productions.");
continue; //for i loop
}
//Check for nested expression - convert to non-terminal
BnfExpression expr = child as BnfExpression;
if (expr != null) {
child = new NonTerminal(null, expr);
elemList[i] = child;
}
CollectTermsRecursive(child);
}//for i
}//method
protected static IEnumerable <T> GetAllAstNodesOf <T>(ParseTreeNode node, BnfTerm term)
{
if (node.Term == term)
{
yield return((T)node.AstNode);
}
foreach (var n in node.ChildNodes)
{
if (n.Term == term)
{
yield return((T)n.AstNode);
}
if (n.ChildNodes.Count == 0)
{
continue;
}
foreach (var result in n.ChildNodes.SelectMany(nInner => GetAllAstNodesOf <T>(nInner, term)))
{
yield return(result);
}
}
}
private void ResolveConflictByHints(ParserState state, BnfTerm conflict)
{
if (TryResolveConflictByHints(state, conflict))
state.BuilderData.ResolvedConflicts.Add(conflict);
}
protected BnfExpression MakeListRule(NonTerminal list, BnfTerm delimiter, BnfTerm listMember, TermListOptions options = TermListOptions.PlusList)
{
//If it is a star-list (allows empty), then we first build plus-list
var isPlusList = !options.IsSet(TermListOptions.AllowEmpty);
var allowTrailingDelim = options.IsSet(TermListOptions.AllowTrailingDelimiter) && delimiter != null;
//"plusList" is the list for which we will construct expression - it is either extra plus-list or original list.
// In the former case (extra plus-list) we will use it later to construct expression for list
NonTerminal plusList = isPlusList ? list : new NonTerminal(listMember.Name + "+");
plusList.SetFlag(TermFlags.IsList);
plusList.Rule = plusList; // rule => list
if (delimiter != null)
plusList.Rule += delimiter; // rule => list + delim
if (options.IsSet(TermListOptions.AddPreferShiftHint))
plusList.Rule += PreferShiftHere(); // rule => list + delim + PreferShiftHere()
plusList.Rule += listMember; // rule => list + delim + PreferShiftHere() + elem
plusList.Rule |= listMember; // rule => list + delim + PreferShiftHere() + elem | elem
if (isPlusList) {
// if we build plus list - we're almost done; plusList == list
// add trailing delimiter if necessary; for star list we'll add it to final expression
if (allowTrailingDelim)
plusList.Rule |= list + delimiter; // rule => list + delim + PreferShiftHere() + elem | elem | list + delim
} else {
// Setup list.Rule using plus-list we just created
list.Rule = Empty | plusList;
if (allowTrailingDelim)
list.Rule |= plusList + delimiter | delimiter;
plusList.SetFlag(TermFlags.NoAstNode);
list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower
}
return list.Rule;
}
public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember)
{
return MakeListRule(listNonTerminal, delimiter, listMember, TermListOptions.StarList);
}
public BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember)
{
return MakeListRule(listNonTerminal, delimiter, listMember);
}
public BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm listMember)
{
return MakeListRule(listNonTerminal, null, listMember);
}
public BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember)
{
listNonTerminal.SetOption(TermOptions.IsList);
if (delimiter == null)
listNonTerminal.Rule = listMember | listNonTerminal + listMember;
else
listNonTerminal.Rule = listMember | listNonTerminal + delimiter + listMember;
return listNonTerminal.Rule;
}
private ValueReader uniOpCreate(BnfTerm op, ValueReader value)
{
if (grammar.Not == op) return new NotOpReader(value);
if (grammar.Minus == op) return new NegOpReader(value);
Game.CurrentGame.Die("Error with implementation of operator " + op.Name);
return null;
}
public static BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) {
bool allowTrailingDelimiter = (options & TermListOptions.AllowTrailingDelimiter) != 0;
if (delimiter == null) {
//it is much simpler case
listNonTerminal.SetFlag(TermFlags.IsList);
listNonTerminal.Rule = _currentGrammar.Empty | listNonTerminal + listMember;
return listNonTerminal.Rule;
}
//Note that deceptively simple version of the star-rule
// Elem* -> Empty | Elem | Elem* + delim + Elem
// does not work when you have delimiters. This simple version allows lists starting with delimiters -
// which is wrong. The correct formula is to first define "Elem+"-list, and then define "Elem*" list
// as "Elem* -> Empty|Elem+"
NonTerminal plusList = new NonTerminal(listMember.Name + "+");
plusList.Rule = MakePlusRule(plusList, delimiter, listMember);
plusList.SetFlag(TermFlags.NoAstNode); //to allow it to have AstNodeType not assigned
if (allowTrailingDelimiter)
listNonTerminal.Rule = _currentGrammar.Empty | plusList | plusList + delimiter;
else
listNonTerminal.Rule = _currentGrammar.Empty | plusList;
listNonTerminal.SetFlag(TermFlags.IsListContainer);
return listNonTerminal.Rule;
}
public static BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm listMember) {
return MakeStarRule(listNonTerminal, null, listMember, TermListOptions.None);
}
public static BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember) {
if (delimiter == null)
listNonTerminal.Rule = listMember | listNonTerminal + listMember;
else
listNonTerminal.Rule = listMember | listNonTerminal + delimiter + listMember;
listNonTerminal.SetFlag(TermFlags.IsList);
return listNonTerminal.Rule;
}
public static BnfExpression MakePlusRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember, TermListOptions options) {
bool allowTrailingDelimiter = (options & TermListOptions.AllowTrailingDelimiter) != 0;
if (delimiter == null || !allowTrailingDelimiter)
return MakePlusRule(listNonTerminal, delimiter, listMember);
//create plus list
var plusList = new NonTerminal(listMember.Name + "+");
plusList.Rule = MakePlusRule(listNonTerminal, delimiter, listMember);
listNonTerminal.Rule = plusList | plusList + delimiter;
listNonTerminal.SetFlag(TermFlags.IsListContainer);
return listNonTerminal.Rule;
}
public BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember)
{
if (delimiter == null) {
//it is much simpler case
listNonTerminal.SetOption(TermOptions.IsList);
listNonTerminal.Rule = Empty | listNonTerminal + listMember;
return listNonTerminal.Rule;
}
NonTerminal tmp = new NonTerminal(listMember.Name + "+");
MakePlusRule(tmp, delimiter, listMember);
listNonTerminal.Rule = Empty | tmp;
listNonTerminal.SetOption(TermOptions.IsStarList);
return listNonTerminal.Rule;
}
}//method
private int GetLookaheadOccurenceCount(ParserState state, BnfTerm lookahead) {
var result = 0;
foreach (var reduceItem in state.BuilderData.ReduceItems)
if (reduceItem.AllLookaheads.Contains(lookahead))
result++;
//add 1 if it is shift term
if (state.BuilderData.ShiftTerms.Contains(lookahead))
result++;
return result;
}
public static BnfExpression MakeStarRule(NonTerminal listNonTerminal, BnfTerm delimiter, BnfTerm listMember) {
if (delimiter == null) {
//it is much simpler case
listNonTerminal.SetOption(TermOptions.IsList);
listNonTerminal.Rule = _currentGrammar.Empty | listNonTerminal + listMember;
return listNonTerminal.Rule;
}
//Note that deceptively simple version of the star-rule
// Elem* -> Empty | Elem | Elem* + delim + Elem
// does not work when you have delimiters. This simple version allows lists starting with delimiters -
// which is wrong. The correct formula is to first define "Elem+"-list, and then define "Elem*" list
// as "Elem* -> Empty|Elem+"
NonTerminal tmp = new NonTerminal(listMember.Name + "+");
tmp.SetOption(TermOptions.IsTransient); //important - mark it as Transient so it will be eliminated from AST tree
MakePlusRule(tmp, delimiter, listMember);
listNonTerminal.Rule = _currentGrammar.Empty | tmp;
//listNonTerminal.SetOption(TermOptions.IsStarList);
return listNonTerminal.Rule;
}
private bool TryResolveConflictByHints(ParserState state, BnfTerm conflict)
{
var stateData = state.BuilderData;
//reduce hints
var reduceItems = stateData.ReduceItems.SelectByLookahead(conflict);
foreach(var reduceItem in reduceItems)
if (reduceItem.Core.Hints != null)
foreach (var hint in reduceItem.Core.Hints)
if (hint.HintType == HintType.ResolveToReduce) {
var newAction = ParserAction.CreateReduce(reduceItem.Core.Production);
state.Actions[conflict] = newAction; //replace/add reduce action
return true;
}
//Shift hints
var shiftItems = stateData.ShiftItems.SelectByCurrent(conflict);
foreach (var shiftItem in shiftItems)
if (shiftItem.Core.Hints != null)
foreach (var hint in shiftItem.Core.Hints)
if (hint.HintType == HintType.ResolveToShift) {
//shift action is already there
return true;
}
//code hints
// first prepare data for conflict action: reduceProduction (for possible reduce) and newState (for possible shift)
var reduceProduction = reduceItems.First().Core.Production; //take first of reduce productions
ParserState newState = (state.Actions.ContainsKey(conflict) ? state.Actions[conflict].NewState : null);
// Get all items that might contain hints; first take all shift items and reduce items in conflict;
// we should also add lookahead sources of reduce items. Lookahead source is an LR item that produces the lookahead,
// so if it contains a code hint right before the lookahead term, then it applies to this conflict as well.
var allItems = new LRItemList();
allItems.AddRange(shiftItems);
foreach (var reduceItem in reduceItems) {
allItems.Add(reduceItem);
allItems.AddRange(reduceItem.ReducedLookaheadSources);
}
// Scan all items and try to find hint with resolution type Code
foreach (var item in allItems)
if (item.Core.Hints != null)
foreach (var hint in item.Core.Hints)
if (hint.HintType == HintType.ResolveInCode) {
//found hint with resolution type "code" - this is instruction to use custom code here to resolve the conflict
// create new ConflictAction and place it into Actions table
var newAction = ParserAction.CreateCodeAction(newState, reduceProduction);
state.Actions[conflict] = newAction; //replace/add reduce action
return true;
}
return false;
}
protected BnfExpression MakeListRule(NonTerminal list, BnfTerm delimiter, BnfTerm listMember, TermListOptions options = TermListOptions.PlusList)
{
//If it is a star-list (allows empty), then we first build plus-list
var isStarList = options.IsSet(TermListOptions.AllowEmpty);
NonTerminal plusList = isStarList ? new NonTerminal(listMember.Name + "+") : list;
//"list" is the real list for which we will construct expression - it is either extra plus-list or original listNonTerminal.
// In the latter case we will use it later to construct expression for listNonTerminal
plusList.Rule = plusList; // rule => list
if (delimiter != null)
plusList.Rule += delimiter; // rule => list + delim
if (options.IsSet(TermListOptions.AddPreferShiftHint))
plusList.Rule += PreferShiftHere(); // rule => list + delim + PreferShiftHere()
plusList.Rule += listMember; // rule => list + delim + PreferShiftHere() + elem
plusList.Rule |= listMember; // rule => list + delim + PreferShiftHere() + elem | elem
//trailing delimiter
if (options.IsSet(TermListOptions.AllowTrailingDelimiter) & delimiter != null)
plusList.Rule |= list + delimiter; // => list + delim + PreferShiftHere() + elem | elem | list + delim
// set Rule value
plusList.SetFlag(TermFlags.IsList);
//If we do not use exra list - we're done, return list.Rule
if (plusList == list)
return list.Rule;
// Let's setup listNonTerminal.Rule using plus-list we just created
//If we are here, TermListOptions.AllowEmpty is set, so we have star-list
list.Rule = Empty | plusList;
plusList.SetFlag(TermFlags.NoAstNode);
list.SetFlag(TermFlags.IsListContainer); //indicates that real list is one level lower
return list.Rule;
}
private bool ResolveConflictByPrecedence(ParserState state, BnfTerm conflict)
{
var stateData = state.BuilderData;
if (!conflict.IsSet(TermOptions.UsePrecedence)) return false;
if (!stateData.ShiftTerms.Contains(conflict)) return false; //it is not shift-reduce
//first find reduce items
var reduceItems = stateData.ReduceItems.SelectByLookahead(conflict);
if (reduceItems.Count > 1) return false; // if it is reduce-reduce conflict, we cannot fix it by precedence
var reduceItem = reduceItems.First();
//remove shift action and replace it with operator action
var oldAction = state.Actions[conflict];
var action = ParserAction.CreateOperator(oldAction.NewState, reduceItem.Core.Production);
state.Actions[conflict] = action;
stateData.ResolvedConflicts.Add(conflict);
return true;
}
internal static BnfExpression Op_Plus(BnfTerm term1, BnfTerm term2)
{
return term1 + term2;
}
public BnfExpression(BnfTerm element)
: this()
{
Data[0].Add(element);
}
private static bool IsMainChild(BnfTerm bnfTerm)
{
return !(bnfTerm is KeyTerm) && !(bnfTerm is GrammarHint);
}