private static void WriteProductions(Grammar grammar, IndentedTextWriter output)
{
output.WriteLine("// Production rules:");
output.WriteLine();
foreach (var prod in grammar.Productions)
{
output.WriteLine("// {0}:", prod.Index);
output.Write("{0} :", prod.Outcome.Name);
if (prod.Pattern.Length == 0)
{
output.Write(" /*empty*/");
}
else
{
foreach (var symb in prod.Pattern)
{
output.Write(' ');
output.Write(symb.Name);
}
}
output.WriteLine(";");
output.WriteLine();
}
}
public void Test()
{
const int tokenCount = 50;
const int ruleSize = 10;
int[] tokens = new int[tokenCount];
this.grammar = new Grammar();
for (int i = 0; i != tokenCount; ++i)
{
tokens[i] = grammar.Symbols.Add(i.ToString()).Index;
}
int iterationCount = tokenCount - ruleSize;
for (int i = 0; i < iterationCount; ++i)
{
int outcome = tokens[i];
int[] pattern = new int[ruleSize];
Array.Copy(tokens, i + 1, pattern, 0, ruleSize);
grammar.Productions.Define(outcome, pattern);
}
var target = new Lalr1Dfa(new GrammarAnalysis(grammar), LrTableOptimizations.Default);
}
private void WriteMatchers(Grammar grammar, IndentedTextWriter output)
{
output.WriteLine("// Scanner Conditions: ");
output.WriteLine();
foreach (var condition in grammar.Conditions)
{
output.WriteLine("condition {0}", condition.Name);
output.WriteLine("{");
++output.Indent;
foreach (var matcher in condition.Matchers)
{
var transition = matcher.NextCondition == null
? ""
: string.Format("{{goto {0}}}", matcher.NextCondition.Name);
output.WriteLine(
"{0} {1}: /{2}/;",
Name(matcher.Outcome),
transition,
matcher.Pattern);
}
--output.Indent;
output.WriteLine("}");
output.WriteLine();
}
}
public void IsNullableTest()
{
var grammar = new Grammar();
var S = grammar.Symbols.Add("S");
var a = grammar.Symbols.Add("a");
var b = grammar.Symbols.Add("b");
var A = grammar.Symbols.Add("A");
var B = grammar.Symbols.Add("B");
grammar.Start = S;
grammar.Productions.Define(S, new[] { b, A });
grammar.Productions.Define(A, new[] { a, A, B });
grammar.Productions.Define(A, new Symbol[0]);
grammar.Productions.Define(B, new Symbol[0]);
var target = new RuntimeGrammar(grammar);
Assert.IsTrue(target.IsNullable(A.Index));
Assert.IsTrue(target.IsNullable(B.Index));
Assert.IsFalse(target.IsNullable(a.Index));
Assert.IsFalse(target.IsNullable(b.Index));
Assert.IsFalse(target.IsNullable(S.Index));
}
public SppfGraphWriter(Grammar grammar, IGraphView graph, bool showRules)
{
this.grammar = grammar;
this.graph = graph;
this.visited = new HashSet<SppfNode>();
this.front = new Stack<SppfNode>();
this.showRules = showRules;
}
public static bool GrammarEquals(Grammar x, Grammar y)
{
return x == y
|| (x != null
&& y != null
&& Enumerable.SequenceEqual(y.Productions, x.Productions, EqualityComparer<Production>.Default)
&& Enumerable.SequenceEqual(y.Symbols, x.Symbols));
}
public RuntimeGrammar(Grammar grammar)
{
this.grammar = grammar;
IRuntimeNullableFirstTables tables = new NullableFirstTables(grammar);
this.isNullable = tables.TokenToNullable;
this.MaxRuleSize = tables.MaxRuleSize;
this.symbolCount = grammar.Symbols.Count;
}
public void DefineAmbiguities(Grammar grammar)
{
foreach (var prod in stateToTokenProducer.Values)
{
var ambSymbol = grammar.Symbols.FindOrAddAmbiguous(prod.MainTokenId, prod.PossibleTokens);
prod.State.EnvelopeId = ambSymbol.Index;
#if DEBUG
Debug.WriteLine("Created ambiguous symbol {0} for state #{1}", ambSymbol, prod.State.Index);
#endif
}
}
public NullableFirstTables(Grammar grammar)
{
this.grammar = grammar;
int count = grammar.Symbols.Count;
this.tokenSet = new BitSetType(count);
this.firsts = new MutableIntSet[count];
this.isNullable = new bool[count];
MaxRuleSize = grammar.Productions.Select(r => r.PatternTokens.Length).Max();
Build();
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(@"blank+"),
new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num),
new Matcher(
@"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
ident),
new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
new Matcher(ScanPattern.CreateLiteral("("), lParen),
new Matcher(ScanPattern.CreateLiteral(")"), rParen),
}
}
}
};
var target = new DfaSimulationLexer(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
collector.Select(msg => msg.Value).ToArray());
}
private static void WriteSummaryComment(Grammar grammar, IndentedTextWriter output)
{
output.WriteLine("/*");
++output.Indent;
output.WriteLine("symbols : {0}", grammar.Symbols.Count);
output.WriteLine("terminals : {0}", grammar.Symbols.Where(s => s.IsTerminal).Count());
output.WriteLine("non-terminals : {0}", grammar.Symbols.Where(s => !s.IsTerminal).Count());
output.WriteLine("productions : {0}", grammar.Productions.Count);
output.WriteLine("mergers : {0}", grammar.Mergers.Count);
output.WriteLine("matchers : {0}", grammar.Matchers.Count);
--output.Indent;
output.WriteLine("*/");
output.WriteLine();
}
public void Test()
{
var originalGrammar = new Grammar();
var red = originalGrammar.Symbols.Add("red");
var green = originalGrammar.Symbols.Add("green", SymbolCategory.ExplicitlyUsed);
var blue = originalGrammar.Symbols.Add("blue");
originalGrammar.Productions.Define(red, new[] { green, blue });
originalGrammar.Productions.Define(blue, new[] { red, green });
originalGrammar.Productions.Define(blue, new Symbol[0]);
originalGrammar.Start = red;
GrammarSerializer target = new GrammarSerializer(originalGrammar);
var factory = new CachedMethod<Func<Grammar>>("GrammarSerializerTest.Assembly0", (emit, args) => { target.Build(emit); return emit.Ret(); }).Delegate;
var recreated = factory();
Assert.IsTrue(GrammarEquals(originalGrammar, recreated));
}
public LrGraph(IReportData data)
{
this.grammar = data.Grammar;
this.automata = data.ParserAutomata;
}
private void WhenGrammarIsInlined()
{
IProductionInliner target = new ProductionInliner(originalGrammar);
this.resultGrammar = target.Inline();
}
public void SetUp()
{
this.originalGrammar = new Grammar();
var symbols = originalGrammar.Symbols;
this.start = symbols.Add("start");
this.prefix = symbols.Add("prefix");
this.suffix = symbols.Add("suffix");
this.term1 = symbols.Add("term1");
this.term2 = symbols.Add("term2");
this.term3 = symbols.Add("term3");
this.term4 = symbols.Add("term4");
this.term5 = symbols.Add("term5");
this.inlinedNonTerm = symbols.Add("inlinedNonTerm");
this.nestedNonTerm = symbols.Add("nestedNonTerm");
originalGrammar.Start = start;
var prod = originalGrammar.Productions.Define(start, new[] { prefix, inlinedNonTerm, suffix });
prod.Actions.Add(new ForeignAction(0, 3));
}
public GrammarAnalysis(Grammar grammar)
{
this.grammar = grammar;
this.tables = new NullableFirstTables(grammar);
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols = {
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
ScanPattern.CreateRegular(
null,
@"[ \t]+"))
{
Joint = { new CilMatcher(typeof(void)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
num)
{
Joint = { new CilMatcher(typeof(Num)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
ident)
{
Joint = { new CilMatcher(typeof(string)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[""](?: \\[""] | [^""])* [""]"),
qStr)
{
Joint = { new CilMatcher(typeof(QStr)) }
},
new Matcher(
ScanPattern.CreateLiteral("("),
lParen),
new Matcher(
ScanPattern.CreateLiteral(")"),
rParen),
}
}
}
};
var target = new BootstrapScanner(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
null,
ExceptionLogging.Instance);
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
collector.Select(msg => msg.Value).ToArray());
}
private static SymbolBase GetMatcherOutcomeSymbol(
Grammar grammar,
ICilSymbolResolver symbolResolver,
CilSymbolRef mainOutcome,
IEnumerable<CilSymbolRef> allOutcomes)
{
Symbol main = symbolResolver.GetSymbol(mainOutcome);
Symbol[] all = (from outcome in allOutcomes
select symbolResolver.GetSymbol(outcome))
.ToArray();
switch (all.Length)
{
case 0: return null;
case 1: return main;
default: return new AmbiguousSymbol(main, all);
}
}
private static Condition ConditionFromType(Grammar grammar, Type type)
{
if (type == null)
{
return null;
}
foreach (var cond in grammar.Conditions)
{
var binding = cond.Joint.The<CilCondition>();
if (binding.ConditionType == type)
{
return cond;
}
}
throw new InvalidOperationException("Undefined condition: " + type.FullName);
}
public TokenIdentitiesSerializer(Grammar grammar)
{
this.grammar = grammar;
}
public void Write(Grammar grammar, TextWriter output)
{
WriteIndented(grammar, new IndentedTextWriter(output, Indent));
}
private static void InitContextProvider(
Grammar grammar,
CilContextProvider cilProvider,
ForeignContextProvider provider)
{
provider.Joint.Add(cilProvider);
foreach (var cilContext in cilProvider.Contexts)
{
ForeignContext context;
if (grammar.Contexts.FindOrAdd(cilContext.UniqueName, out context))
{
context.Joint.Add(cilContext);
}
provider.Add(context);
}
}
public void Test()
{
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var begin = new Symbol("begin");
var end = new Symbol("end");
var assign = new Symbol(":=");
var grammar = new Grammar
{
Symbols =
{
num,
ident,
qStr,
begin,
end,
assign
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
@"digit+ ('.' digit+)? | '.' digit+",
num),
new Matcher(
@"[01234567]+ 'Q'",
num),
new Matcher(
@"alpha alnum*",
ident),
new Matcher(
@"
quot
~(quot | esc)*
(esc . ~(quot | esc)* )*
quot
",
qStr),
new Matcher(
ScanPattern.CreateLiteral("begin"),
begin),
new Matcher(
ScanPattern.CreateLiteral("end"),
end),
new Matcher(
ScanPattern.CreateLiteral(":="),
assign),
new Matcher(
@"blank+"),
}
}
}
};
var target = new TdfaSimulationLexer(
"b:=10Q \"foo\"",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "b", ":=", "10Q", "\"foo\"" },
collector.Select(msg => msg.Value).ToArray());
}
private static Grammar BuildGrammar(CilGrammar definition)
{
var result = new Grammar();
InitContextProvider(
result,
definition.GlobalContextProvider,
result.GlobalContextProvider);
var symbolResolver = definition.SymbolResolver;
// Define grammar tokens
foreach (var cilSymbol in symbolResolver.Definitions)
{
Symbol symbol;
if (cilSymbol.Type == typeof(Exception))
{
cilSymbol.Symbol = symbol = (Symbol)result.Symbols[PredefinedTokens.Error];
symbol.Joint.Add(
new CilSymbol
{
Type = typeof(Exception),
Symbol = symbol,
Categories = SymbolCategory.DoNotDelete | SymbolCategory.DoNotInsert
});
}
else
{
symbol = new Symbol(cilSymbol.Name)
{
Categories = cilSymbol.Categories,
Joint = { cilSymbol }
};
result.Symbols.Add(symbol);
cilSymbol.Symbol = symbol;
}
}
foreach (CilSymbolFeature<Precedence> feature in definition.Precedence)
{
var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
symbol.Precedence = feature.Value;
}
foreach (CilSymbolFeature<CilContextProvider> feature in definition.LocalContextProviders)
{
var symbol = symbolResolver.GetSymbol(feature.SymbolRef);
if (symbol != null)
{
InitContextProvider(result, feature.Value, symbol.LocalContextProvider);
}
}
result.Start = symbolResolver.GetSymbol(definition.Start);
// Define grammar rules
foreach (var cilProduction in definition.Productions)
{
Symbol outcome = symbolResolver.GetSymbol(cilProduction.Outcome);
var pattern = Array.ConvertAll(cilProduction.Pattern, symbolResolver.GetSymbol);
// Try to find existing rules whith same token-signature
Production production;
if (result.Productions.FindOrAdd(outcome, pattern, out production))
{
ForeignContextRef contextRef = CreateActionContextRef(cilProduction.Context);
production.Actions.Add(new ForeignAction(pattern.Length, contextRef));
}
var action = production.Actions[0];
action.Joint.Add(cilProduction);
production.ExplicitPrecedence = cilProduction.Precedence;
}
// Create conditions to allow referencing them from matchers
foreach (CilCondition cilCondition in definition.Conditions)
{
var cond = CreateCondtion(result, cilCondition);
result.Conditions.Add(cond);
}
// Create matchers
foreach (CilCondition cilCondition in definition.Conditions)
{
var condition = ConditionFromType(result, cilCondition.ConditionType);
foreach (var cilMatcher in cilCondition.Matchers)
{
SymbolBase outcome = GetMatcherOutcomeSymbol(result, symbolResolver, cilMatcher.MainOutcome, cilMatcher.AllOutcomes);
var matcher = new Matcher(
cilMatcher.Pattern,
outcome,
#if false
context: CreateActionContextRef(cilMatcher.Context),
#endif
nextCondition: ConditionFromType(result, cilMatcher.NextConditionType),
disambiguation: cilMatcher.Disambiguation);
matcher.Joint.Add(cilMatcher);
condition.Matchers.Add(matcher);
}
}
foreach (var cilMerger in definition.Mergers)
{
var symbol = symbolResolver.GetSymbol(cilMerger.Symbol);
var merger = new Merger(symbol) { Joint = { cilMerger } };
result.Mergers.Add(merger);
}
foreach (var report in definition.Reports)
{
result.Reports.Add(report);
}
return result;
}
private static Condition CreateCondtion(Grammar grammar, CilCondition cilCondition)
{
var result = new Condition(cilCondition.ConditionType.FullName)
{
Joint = { cilCondition }
};
InitContextProvider(grammar, cilCondition.ContextProvider, result.ContextProvider);
return result;
}
private void WriteIndented(Grammar grammar, IndentedTextWriter output)
{
WriteSummaryComment(grammar, output);
WriteProductions(grammar, output);
WriteMatchers(grammar, output);
}