private static void TraversePlay()
{
var g = new Grammar(new List <Production> {
CFGParser.Production("<A> → <B>"),
CFGParser.Production("<B> → <A>"),
CFGParser.Production("<B> → 'x'"),
}, Nonterminal.Of("A"));
g = IdentityActions.Annotate(g);
var earley2 = new EarleyParser2(g);
var sentence = Sentence.FromWords("x");
var sppf2 = earley2.ParseGetForest(sentence);
DotRunner.Run(DotBuilder.GetRawDot(sppf2), "infinite");
var t2 = new Traversal(sppf2, g);
var r2 = t2.Traverse();
//foreach (var option in r2) {
// var s2 = (Sentence)option.Payload;
// if (!sentence.SequenceEqual(s2)) {
// throw new Exception();
// }
//}
}
internal bool PredictedAlreadyAndSet(Nonterminal nonterminal)
{
var predicted = _nonterminalsPredicted.Contains(nonterminal);
_nonterminalsPredicted.Add(nonterminal);
return(predicted);
}
static SimpleMaths()
{
var number = Terminals.Double("Number");
var expression = Nonterminal.Create <double>("Expression");
expression.SetProductions(
number.AsIs(),
expression.Extended().Append("+").Extend(expression).Finish((x1, x2) => x1 + x2),
expression.Extended().Append("-").Extend(expression).Finish((x1, x2) => x1 - x2),
expression.Extended().Append("*").Extend(expression).Finish((x1, x2) => x1 * x2),
expression.Extended().Append("/").Extend(expression).Finish((x1, x2) => x1 / x2),
"-".Appended().Extend(expression).WithPrecedence(out var NEG).Finish(x => - x),
expression.Extended().Append("^").Extend(expression).Finish(Math.Pow),
"(".Appended().Extend(expression).Append(")").AsIs());
var opScope = new OperatorScope(
new LeftAssociative("+", "-"),
new LeftAssociative("*", "/"),
new PrecedenceOnly(NEG),
new RightAssociative("^"));
Designtime = expression.WithOperatorScope(opScope);
Runtime = Designtime.Build();
}
/// <summary>
/// The point of this is to evaluate all of the expressions and assign them to the rule (NonTerminal). The
/// "activeMethod" business is to escape recursion if a rule calls itself.
/// </summary>
private void InvocationHandler(Invocation invocation)
{
Nonterminal result;
if (rules.TryGetValue(invocation.Method.Name, out result) && result.Expression != null)
{
invocation.ReturnValue = result;
}
else
{
if (activeMethod != null)
{
Nonterminal rule = rules[invocation.Method.Name];
invocation.ReturnValue = rule;
}
else
{
activeMethod = invocation.Method;
invocation.Proceed();
Nonterminal rule = rules[invocation.Method.Name];
if (rule.Expression == null)
{
rule.Expression = (Expression)invocation.ReturnValue;
}
invocation.ReturnValue = rule;
// Now check for captured nonterminals (i.e. .Capture("foo"))
var walker = new CaptureWalker();
rule.Expression.Accept(walker, this);
activeMethod = null;
}
}
}
/// <summary>
/// Productions MUST BE:
/// - left factored
/// - without left recursions
/// </summary>
/// <param name="productions">Processed productions</param>
public void FillByProcessedProductions(IEnumerable <Production> productions, Nonterminal axiom)
{
// A -> alpha
foreach (var production in productions)
{
var useProductionAction = AutomateAction.FromProduction(production);
// FIRST(alpha)
var first = Helper.First(productions, production.RightPart);
foreach (var terminal in first)
{
this[production.LeftPart, terminal] = useProductionAction;
}
if (first.Contains(GeneralizedTerminal.Epsilon))
{
// FOLLOW(A)
var follow = Helper.Follow(productions, production.LeftPart, axiom);
foreach (var terminal in follow)
{
this[production.LeftPart, terminal] = useProductionAction;
}
}
}
}
public AutomateAction this[Nonterminal nt, Terminal t]
{
get
{
if (_table.ContainsKey(nt))
{
foreach (var item in _table[nt])
{
if (item.Key.IsAppropriateTerminal(t))
{
return(item.Value);
}
}
}
return(null);
}
set
{
if (!_table.ContainsKey(nt))
{
_table.Add(nt, new Dictionary <Terminal, AutomateAction>());
}
_table[nt][t] = value;
}
}
public void TestEarleys01()
{
ExecuteTest(new Grammar(new List <Production> {
CFGParser.Production("<X_0> → <X_5> 'x5' [66.743324284322242]"),
CFGParser.Production("<X_5> → 'x6' 'x6' [18.445467280897063]")
}, Nonterminal.Of("X_0")), "x6");
}
private static Production generateRandomProduction(Nonterminal lhs, List <Nonterminal> nts, double epsylonProb = 0.2, double nontermProb = 0.4, double lengthIncreaseProb = 0.5)
{
List <GrammarSymbol> rhs = new List <GrammarSymbol>();
//epsylon
if (rnd.NextDouble() < epsylonProb)
{
return(new Production(lhs, rhs.ToArray()));
}
do
{
if (rnd.NextDouble() < nontermProb)
{
rhs.Add(nts[rnd.Next(nts.Count)]); // --> nonterminal
}
else
{ // --> terminal
char term = (char)('a' + rnd.Next(10));
rhs.Add(new Exprinal <String>("" + term, "" + term));
}
} while (rnd.NextDouble() < lengthIncreaseProb); //length of right hand side
return(new Production(lhs, rhs.ToArray()));
}
public static ContextFreeGrammar GenerateRandom()
{
Nonterminal startS = new Nonterminal("S");
List <Nonterminal> nts = new List <Nonterminal>();
nts.Add(startS);
//nonterminals
int ntcount = rnd.Next(2, 6);
while (nts.Count < ntcount)
{
var newNt = new Nonterminal("" + (char)('A' + rnd.Next(26))); //random NT name
if (!nts.Contains(newNt))
{
nts.Add(newNt);
}
}
//productions
List <Production> prods = new List <Production>();
prods.Add(generateRandomProduction(startS, nts, 0, 1, 0.45)); //add a production for Start with only NTs
foreach (Nonterminal nt in nts)
{
prods.Add(generateRandomProduction(nt, nts)); //at least 1 prod per NT
while (rnd.NextDouble() < 0.5)
{
prods.Add(generateRandomProduction(nt, nts)); //generate more prods
}
}
return(new ContextFreeGrammar(startS, prods));
}
/// <summary>
/// Returns a Nonterminal that references the current expression. Nonterminals are
/// the only structure that retains structural meaning after parsing (i.e. it
/// has representation in the output stream / CST.) Thus, if you ever need to do
/// something to your parse result by referring to elements in your pattern,
/// this is how you can solidify them.
/// </summary>
public static Nonterminal Capture(this Expression expression)
{
Nonterminal result = new Nonterminal();
result.Expression = expression;
return(result);
}
public void TestAccepts03()
{
var productions = new HashSet <Production> {
CFGParser.Production("<S> -> <S> <S> <S>"),
CFGParser.Production("<S> -> ε"),
CFGParser.Production("<S> -> 'x2' 'x0'"),
CFGParser.Production("<S> -> 'x0' <S>"),
CFGParser.Production("<S> -> 'x4'"),
CFGParser.Production("<S> -> <S> 'x0' 'x4'"),
CFGParser.Production("<S> -> 'x3' <S> 'x3' <S> 'x0'"),
CFGParser.Production("<S> -> 'x2' <S> <S> <S> <S>"),
CFGParser.Production("<S> -> <S> <S> <S> <S> <S>"),
CFGParser.Production("<S> -> 'x0' <S> <S>"),
CFGParser.Production("<S> -> <S>"),
CFGParser.Production("<S> -> 'x0' <S> <S> 'x1'"),
CFGParser.Production("<S> -> 'x3' 'x2' 'x1'"),
CFGParser.Production("<S> -> 'x0' 'x0' 'x2'"),
};
Grammar g = new Grammar(productions, Nonterminal.Of("S"));
CNFGrammar h = g.ToCNF();
Assert.IsTrue(h.Accepts(Sentence.FromLetters("")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x4")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x4 x0 x4")));
}
private void BubblingProcessNode(SyntaxTreeNode node)
{
if (node.Value is Nonterminal)
{
Nonterminal nonterminal = node.Value as Nonterminal;
if (nonterminal.Equals(MyNonterminals.ASSIGN_STATEMENT) && node.ChildrenCount == 4)
{
var idToken = (node[1].Value as ConcreteTerminal).Token;
string defVariableName = idToken.Value;
if (!_defindedVariables.Contains(defVariableName))
{
_defindedVariables.Add(defVariableName);
}
else
{
_errors.Add(new Error(idToken, ErrorKind.Semantic, "Variable redefinition"));
}
}
}
else
{
ConcreteTerminal terminal = node.Value as ConcreteTerminal;
if (terminal.Token.HasType(TokenType.Identifier) && !_defindedVariables.Contains(terminal.Token.Value))
{
if (((Nonterminal)node.Parent.Value).Equals(MyNonterminals.FACTOR))
{
_errors.Add(new Error(terminal.Token, ErrorKind.Semantic, "Undefined variable"));
}
}
}
}
public void ParserONE_AND_MORE_OR__ASSIGN_OP__VAR()
{
Nonterminal expr = new Nonterminal(OR, "ASSIGN_OP", "VAR");
Nonterminal lang = new Nonterminal(ONE_AND_MORE, expr);
CheckTest(Resources.ParserONE_AND_MORE_OR__ASSIGN_OP__VAR, true, 21, new ParserLang(lang));
}
public void TestTraversal01()
{
ExecuteTest(new Grammar(new List <Production> {
CFGParser.Production("<X_0> → ε [115.49728913674936]"),
CFGParser.Production("<X_0> → 'x0' 'x0' <X_0> 'x0' 'x0' [32.857227595456521]")
}, Nonterminal.Of("X_0")), "ε");
}
/// <summary>
/// Converts the given cfg into a new one in 2NF, that means that every production has at most 2 symbols on the right hand side
/// </summary>
/// <param name="cfg">cfg</param>
/// <returns>cfg in 2NF</returns>
public static ContextFreeGrammar To2NF(ContextFreeGrammar cfg)
{
var productions = new List<Production>();
int currentNumber = 0;
var productionsToShorten = new Queue<Production>(cfg.GetProductions());
while (productionsToShorten.Count() > 0)
{
var next = productionsToShorten.Dequeue();
if (next.Rhs.Length <= 2)
{
productions.Add(next);
}
else
{
var newNonTerminal = new Nonterminal(PrefixOfAddedNonterminals + currentNumber);
currentNumber++;
Assertion.Assert(!cfg.Variables.Any(v => v.Name.Equals(newNonTerminal.Name)),
string.Format("The nonterminal with id {0} already existed. Please ensure, that the CFG does not use ints as ids", newNonTerminal.Name));
productions.Add(new Production(next.Lhs, next.Rhs[0], newNonTerminal));
productionsToShorten.Enqueue(new Production(newNonTerminal, next.Rhs.Skip(1).ToArray()));
}
}
return new ContextFreeGrammar(cfg.StartSymbol, productions);
}
public void TestTraversal05()
{
ExecuteTest(new Grammar(new List <Production> {
CFGParser.Production("<S> → <S> <S>"),
CFGParser.Production("<S> → 'x'"),
}, Nonterminal.Of("S")), "x x x x x x x x x x x x x x x x");
}
public void TestTraversalAddition()
{
ExecuteTest(new Grammar(new List <Production> {
CFGParser.Production("<S> → <S> '+' <S>"),
CFGParser.Production("<S> → '1'")
}, Nonterminal.Of("S")), "1 + 1 + 1");
}
public void TestParsing18()
{
var g = new Grammar(new List <Production> {
CFGParser.Production("<X_9> → 'x3' <X_4> <X_9> [69.71867415901211]"),
CFGParser.Production("<X_6> → 'x4' [43.169519673180545]"),
CFGParser.Production("<X_0> → 'x0' 'x3' <X_6> <X_9> <X_9> [95.5660355475573]"),
CFGParser.Production("<X_5> → <X_9> 'x1' 'x0' 'x1' 'x3' <X_2> [35.638882444537657]"),
CFGParser.Production("<X_1> → 'x4' 'x3' 'x1' 'x1' <X_9> <X_8> [60.963767072169006]"),
CFGParser.Production("<X_9> → <X_6> [96.869668710916145]"),
CFGParser.Production("<X_8> → 'x1' <X_0> 'x0' <X_2> <X_2> [10.412202848779131]"),
CFGParser.Production("<X_4> → ε [89.394112460498746]"),
CFGParser.Production("<X_4> → <X_8> 'x2' <X_5> 'x1' [41.46934854261081]"),
CFGParser.Production("<X_2> → ε [28.04076097674703]"),
CFGParser.Production("<X_8> → ε [55.798571558109757]"),
CFGParser.Production("<X_0> → 'x2' 'x2' 'x3' <X_6> [48.407048357374521]"),
CFGParser.Production("<X_0> → <X_1> 'x3' 'x2' [82.3935272774629]"),
CFGParser.Production("<X_1> → <X_8> <X_1> <X_2> [68.051246746932733]")
}, Nonterminal.Of("X_0"));
var sentences = new List <Sentence>();
sentences.Add(Sentence.FromWords("x3 x2"));
ExecuteTest(g, sentences);
}
private Nonterminal Map(NonterminalExpr rule)
{
if (!Nonterminals.TryGetValue(rule, out var mapped))
{
mapped = new Nonterminal(rule.Name);
Nonterminals.Add(rule, mapped);
foreach (var chain in rule.Body)
{
var body = new List <Symbol>();
foreach (var symbol in chain)
{
switch (symbol)
{
case TerminalExpr terminal:
body.Add(Map(terminal));
break;
case NonterminalExpr nonterminal:
body.Add(Map(nonterminal));
break;
}
}
Productions.Add(new Production(mapped, body));
}
}
return(mapped);
}
public void TestProduction()
{
var actualp = CFGParser.Production(@"<S> -> <X> 'a' <S> [3.0]");
var expectedp = new Production(
Nonterminal.Of("S"),
new Sentence {
Nonterminal.Of("X"),
Terminal.Of("a"),
Nonterminal.Of("S")
},
3.0
);
var unexpectedp = new Production(
Nonterminal.Of("S"),
new Sentence {
Terminal.Of("a"),
Nonterminal.Of("X"),
Nonterminal.Of("S")
},
3.0
);
Assert.IsTrue(actualp.ValueEquals(expectedp));
Assert.IsFalse(actualp.ValueEquals(unexpectedp));
}
public void TestAccepts05()
{
var productions = new HashSet <Production> {
CFGParser.Production("<X_2> -> <X_1> <X_0>"),
CFGParser.Production("<X_1> -> <X_2> <X_1> <X_3> 'x2'"),
CFGParser.Production("<X_3> -> <X_0> <X_0> <X_1> <X_3>"),
CFGParser.Production("<X_3> -> ε"),
CFGParser.Production("<X_2> -> <X_0> <X_1> <X_3> <X_1> <X_3>"),
CFGParser.Production("<X_2> -> <X_1> <X_2> <X_2> <X_0>"),
CFGParser.Production("<X_0> -> <X_3> 'x3'"),
CFGParser.Production("<X_2> -> ε"),
CFGParser.Production("<X_0> -> <X_2> <X_1>"),
CFGParser.Production("<X_2> -> <X_0> <X_1> <X_2>"),
CFGParser.Production("<X_1> -> <X_3> <X_3>"),
CFGParser.Production("<X_3> -> 'x3' 'x4'"),
CFGParser.Production("<X_3> -> <X_3> 'x4'"),
CFGParser.Production("<X_1> -> 'x0' 'x4' 'x0' <X_2> <X_0>"),
};
var g = new Grammar(productions, Nonterminal.Of("X_0"));
var h = g.ToCNF();
Assert.IsTrue(h.Accepts(Sentence.FromWords("")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x3")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x3 x4 x3")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x4 x3")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x3 x4 x4 x3")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x3 x4")));
Assert.IsTrue(h.Accepts(Sentence.FromWords("x3 x4 x3 x4")));
}
public void TestParsing15()
{
// S -> aSa | bSb | ε
var g = new Grammar(new List <Production> {
CFGParser.Production(@"<S> -> 'a' <S> 'a'"),
CFGParser.Production(@"<S> -> 'b' <S> 'b'"),
CFGParser.Production(@"<S> -> ε"),
}, Nonterminal.Of("S"));
var sentences = new List <Sentence> {
Sentence.FromLetters("ab"),
Sentence.FromLetters("abc"),
Sentence.FromLetters("aaa"),
Sentence.FromLetters("abbba"),
Sentence.FromLetters(""),
Sentence.FromLetters("aa"),
Sentence.FromLetters("bb"),
Sentence.FromLetters("abba"),
Sentence.FromLetters("baab"),
Sentence.FromLetters("aaaa"),
Sentence.FromLetters("bbbb"),
Sentence.FromLetters("aaabbabbabbaaa"),
};
ExecuteTest(g, sentences);
}
private void InvocationHandler(Invocation invocation)
{
Nonterminal result;
if (rules.TryGetValue(invocation.Method.Name, out result) && result.Expression != null)
{
invocation.ReturnValue = result;
}
else
{
if (activeMethod != null)
{
Nonterminal rule = rules[invocation.Method.Name];
invocation.ReturnValue = rule;
}
else
{
activeMethod = invocation.Method;
invocation.Proceed();
Nonterminal rule = rules[invocation.Method.Name];
if (rule.Expression == null)
{
rule.Expression = (Expression)invocation.ReturnValue;
}
invocation.ReturnValue = rule;
activeMethod = null;
}
}
}
private string GenerateErrorMessage(Nonterminal disclosingNonterminal, ConcreteTerminal inputTerminal)
{
var expected = GetExpectedTerminalsFor(disclosingNonterminal)
.Except(new[] { GeneralizedTerminal.Epsilon, GeneralizedTerminal.EndOfText });
//return string.Format("{0} expected, but received {1} (disclosing: {2})", string.Join(" or ", expected), inputTerminal, disclosingNonterminal);
return(string.Format("{0} expected, but received {1} (start with one of {{{2}}})", disclosingNonterminal, inputTerminal, string.Join(", ", expected)));
}
public IList <EarleyItem> ItemsAtNonterminal(Nonterminal nonterminal)
{
if (!_nonterminalCache.TryGetValue(nonterminal, out var list))
{
return(null);
}
return(list);
}
public LeftRecursion(Nonterminal rule, LeftRecursion next, Grammar grammar)
{
Rule = rule;
InvolvedSet = new BooleanSet(grammar.Nonterminals.Count);
Next = next;
EvalSet = next.EvalSet.Copy();
EvalSet.Add(next.Rule.Index);
}
internal ForestInternal(InteriorNode node, Nonterminal nonterminal) : base(node.StartPosition, node.EndPosition) {
_node = node;
_nonterminal = nonterminal;
//_nodeLookup = new Dictionary<InteriorNode, ForestInternal>();
//_nodeLookup[node] = this;
_options = ForestOption.BuildOptions(node.Families, node.StartPosition, node.EndPosition);
}
public void TestGrammar1Letter()
{
TestGrammar1 grammar = TestGrammar1.Create();
Nonterminal nonterminal = grammar.GetNonterminal(o => o.LetterA());
Assert.AreEqual("LetterA", nonterminal.Name);
Assert.AreEqual('a', ((CharacterTerminal)nonterminal.Expression).Character);
}
public LeftRecursion(Nonterminal rule, Grammar grammar)
{
Rule = rule;
int size = grammar.Nonterminals.Count;
InvolvedSet = new BooleanSet(size);
EvalSet = new BooleanSet(size);
}
/// <summary>
/// Returns a Nonterminal that references the current expression, giving it the specified
/// name. Nonterminals are the only structure that retains structural meaning after
/// parsing (i.e. it has representation in the output stream / CST.) Thus, if you ever
/// need to do something to your parse result by referring to elements in your pattern,
/// this is how you can solidify them.
/// </summary>
public static Nonterminal Capture(this Expression expression, string name)
{
Nonterminal result = new Nonterminal();
result.Name = name;
result.Expression = expression;
return(result);
}
private void BuildList(PropertyInfo property, object astNode, CstNonterminalNode cstNode)
{
ConsumeExpression[] expressions = astAttributes[cstNode.Nonterminal].Select(o => o.GetExpression()).ToArray();
Expression expression = cstNode.Nonterminal.Expression;
if (expressions.Length > 0 || (expression is Sequence || expression is ZeroOrMore || expression is OneOrMore))
{
IEnumerable <ICstNonterminalNode> itemNodes;
if (expressions.Length > 0)
{
itemNodes = GetChildrenByAstExpression(cstNode, expressions);
}
else
{
Func <Expression, Nonterminal> findNonterminal = null;
findNonterminal = current =>
{
if (current is Nonterminal)
{
return((Nonterminal)current);
}
else if (current is Sequence)
{
return(findNonterminal(((Sequence)current).Expressions.First()));
}
else if (current is ZeroOrMore)
{
return(findNonterminal(((ZeroOrMore)current).Operand));
}
else
{
return(findNonterminal(((OneOrMore)current).Operand));
}
};
Nonterminal nonterminal = findNonterminal(expression);
itemNodes = GetChildrenByNonterminal(cstNode, nonterminal);
}
IList list = (IList)Activator.CreateInstance(property.PropertyType);
Type listType = property.PropertyType.GetListElementType();
foreach (var childNode in itemNodes)
{
object listItemAstNode = Build(listType, childNode);
list.Add(listItemAstNode);
}
property.SetValue(astNode, list, null);
}
else
{
IList list = (IList)Activator.CreateInstance(property.PropertyType);
Type listType = property.PropertyType.GetListElementType();
object listItemAstNode = Build(listType, cstNode);
list.Add(listItemAstNode);
property.SetValue(astNode, list, null);
}
}
public StateSet(Nonterminal predictionSeedNonterminal = null) {
_items = new List<Item>();
_seenItems = new Dictionary<Item, Item>(new ItemComparer());
_nonterminalsPredicted = new HashSet<Nonterminal>();
if (predictionSeedNonterminal != null) {
_nonterminalsPredicted.Add(predictionSeedNonterminal);
}
_magicItems = new List<Item>();
}
public SchemeGrammar()
{
var _expression = new Nonterminal("Expression");
var _variable = new Nonterminal("Variable");
var _literal = new Nonterminal("Literal");
var _procedureCall = new Nonterminal("ProcedureCall");
var _lambdaExpression = new Nonterminal("LambdaExpression");
var _conditional = new Nonterminal("Conditional");
var _assignment = new Nonterminal("Assignment");
var _derivedExpression = new Nonterminal("DerivedExpression");
var _macroUse = new Nonterminal("MacroUse");
var _macroBlock = new Nonterminal("MacroBlock");
var _quotation = new Nonterminal("Quotation");
var _selfEvaluating = new Nonterminal("Self-Evaluating");
var _boolean = new Nonterminal("Boolean");
var _number = new Nonterminal("Number");
var _character = new Nonterminal("Character");
var _string = new Nonterminal("String");
var _datum = new Nonterminal("Datum");
_expression.Rule = _variable | _literal | _procedureCall | _lambdaExpression | _conditional | _assignment
| _derivedExpression | _macroUse | _macroBlock;
_literal.Rule = _quotation | _selfEvaluating;
_selfEvaluating.Rule = _boolean | _number | _character | _string;
_quotation.Rule = Key("'") + _datum
| Key("(quote") + _datum;
_root = null;
throw new NotImplementedException();
}
internal bool PredictedAlreadyAndSet(Nonterminal nonterminal) {
var predicted = _nonterminalsPredicted.Contains(nonterminal);
_nonterminalsPredicted.Add(nonterminal);
return predicted;
}
public SymbolNode(Nonterminal symbol, int start, int end) : base(start, end) {
Symbol = symbol;
}
