private static void WriteProductions(Grammar grammar, IndentedTextWriter output) { output.WriteLine("// Production rules:"); output.WriteLine(); foreach (var prod in grammar.Productions) { output.WriteLine("// {0}:", prod.Index); output.Write("{0} :", prod.Outcome.Name); if (prod.Pattern.Length == 0) { output.Write(" /*empty*/"); } else { foreach (var symb in prod.Pattern) { output.Write(' '); output.Write(symb.Name); } } output.WriteLine(";"); output.WriteLine(); } }
public void Test() { const int tokenCount = 50; const int ruleSize = 10; int[] tokens = new int[tokenCount]; this.grammar = new Grammar(); for (int i = 0; i != tokenCount; ++i) { tokens[i] = grammar.Symbols.Add(i.ToString()).Index; } int iterationCount = tokenCount - ruleSize; for (int i = 0; i < iterationCount; ++i) { int outcome = tokens[i]; int[] pattern = new int[ruleSize]; Array.Copy(tokens, i + 1, pattern, 0, ruleSize); grammar.Productions.Define(outcome, pattern); } var target = new Lalr1Dfa(new GrammarAnalysis(grammar), LrTableOptimizations.Default); }
private void WriteMatchers(Grammar grammar, IndentedTextWriter output) { output.WriteLine("// Scanner Conditions: "); output.WriteLine(); foreach (var condition in grammar.Conditions) { output.WriteLine("condition {0}", condition.Name); output.WriteLine("{"); ++output.Indent; foreach (var matcher in condition.Matchers) { var transition = matcher.NextCondition == null ? "" : string.Format("{{goto {0}}}", matcher.NextCondition.Name); output.WriteLine( "{0} {1}: /{2}/;", Name(matcher.Outcome), transition, matcher.Pattern); } --output.Indent; output.WriteLine("}"); output.WriteLine(); } }
public void IsNullableTest() { var grammar = new Grammar(); var S = grammar.Symbols.Add("S"); var a = grammar.Symbols.Add("a"); var b = grammar.Symbols.Add("b"); var A = grammar.Symbols.Add("A"); var B = grammar.Symbols.Add("B"); grammar.Start = S; grammar.Productions.Define(S, new[] { b, A }); grammar.Productions.Define(A, new[] { a, A, B }); grammar.Productions.Define(A, new Symbol[0]); grammar.Productions.Define(B, new Symbol[0]); var target = new RuntimeGrammar(grammar); Assert.IsTrue(target.IsNullable(A.Index)); Assert.IsTrue(target.IsNullable(B.Index)); Assert.IsFalse(target.IsNullable(a.Index)); Assert.IsFalse(target.IsNullable(b.Index)); Assert.IsFalse(target.IsNullable(S.Index)); }
public SppfGraphWriter(Grammar grammar, IGraphView graph, bool showRules) { this.grammar = grammar; this.graph = graph; this.visited = new HashSet<SppfNode>(); this.front = new Stack<SppfNode>(); this.showRules = showRules; }
public static bool GrammarEquals(Grammar x, Grammar y) { return x == y || (x != null && y != null && Enumerable.SequenceEqual(y.Productions, x.Productions, EqualityComparer<Production>.Default) && Enumerable.SequenceEqual(y.Symbols, x.Symbols)); }
public RuntimeGrammar(Grammar grammar) { this.grammar = grammar; IRuntimeNullableFirstTables tables = new NullableFirstTables(grammar); this.isNullable = tables.TokenToNullable; this.MaxRuleSize = tables.MaxRuleSize; this.symbolCount = grammar.Symbols.Count; }
public void DefineAmbiguities(Grammar grammar) { foreach (var prod in stateToTokenProducer.Values) { var ambSymbol = grammar.Symbols.FindOrAddAmbiguous(prod.MainTokenId, prod.PossibleTokens); prod.State.EnvelopeId = ambSymbol.Index; #if DEBUG Debug.WriteLine("Created ambiguous symbol {0} for state #{1}", ambSymbol, prod.State.Index); #endif } }
public NullableFirstTables(Grammar grammar) { this.grammar = grammar; int count = grammar.Symbols.Count; this.tokenSet = new BitSetType(count); this.firsts = new MutableIntSet[count]; this.isNullable = new bool[count]; MaxRuleSize = grammar.Productions.Select(r => r.PatternTokens.Length).Max(); Build(); }
public void Test() { var lParen = new Symbol("("); var rParen = new Symbol(")"); var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var grammar = new Grammar { Symbols = { lParen, rParen, num, ident, qStr }, Conditions = { new Condition("main") { Matchers = { new Matcher(@"blank+"), new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num), new Matcher( @"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*", ident), new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr), new Matcher(ScanPattern.CreateLiteral("("), lParen), new Matcher(ScanPattern.CreateLiteral(")"), rParen), } } } }; var target = new DfaSimulationLexer( " (1 (\"bar\" +))", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance)); var collector = new Collector<Msg>(); target.Accept(collector); Assert.AreEqual( new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index }, collector.Select(msg => msg.Id).ToArray()); Assert.AreEqual( new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" }, collector.Select(msg => msg.Value).ToArray()); }
private static void WriteSummaryComment(Grammar grammar, IndentedTextWriter output) { output.WriteLine("/*"); ++output.Indent; output.WriteLine("symbols : {0}", grammar.Symbols.Count); output.WriteLine("terminals : {0}", grammar.Symbols.Where(s => s.IsTerminal).Count()); output.WriteLine("non-terminals : {0}", grammar.Symbols.Where(s => !s.IsTerminal).Count()); output.WriteLine("productions : {0}", grammar.Productions.Count); output.WriteLine("mergers : {0}", grammar.Mergers.Count); output.WriteLine("matchers : {0}", grammar.Matchers.Count); --output.Indent; output.WriteLine("*/"); output.WriteLine(); }
public void Test() { var originalGrammar = new Grammar(); var red = originalGrammar.Symbols.Add("red"); var green = originalGrammar.Symbols.Add("green", SymbolCategory.ExplicitlyUsed); var blue = originalGrammar.Symbols.Add("blue"); originalGrammar.Productions.Define(red, new[] { green, blue }); originalGrammar.Productions.Define(blue, new[] { red, green }); originalGrammar.Productions.Define(blue, new Symbol[0]); originalGrammar.Start = red; GrammarSerializer target = new GrammarSerializer(originalGrammar); var factory = new CachedMethod<Func<Grammar>>("GrammarSerializerTest.Assembly0", (emit, args) => { target.Build(emit); return emit.Ret(); }).Delegate; var recreated = factory(); Assert.IsTrue(GrammarEquals(originalGrammar, recreated)); }
public LrGraph(IReportData data) { this.grammar = data.Grammar; this.automata = data.ParserAutomata; }
private void WhenGrammarIsInlined() { IProductionInliner target = new ProductionInliner(originalGrammar); this.resultGrammar = target.Inline(); }
public void SetUp() { this.originalGrammar = new Grammar(); var symbols = originalGrammar.Symbols; this.start = symbols.Add("start"); this.prefix = symbols.Add("prefix"); this.suffix = symbols.Add("suffix"); this.term1 = symbols.Add("term1"); this.term2 = symbols.Add("term2"); this.term3 = symbols.Add("term3"); this.term4 = symbols.Add("term4"); this.term5 = symbols.Add("term5"); this.inlinedNonTerm = symbols.Add("inlinedNonTerm"); this.nestedNonTerm = symbols.Add("nestedNonTerm"); originalGrammar.Start = start; var prod = originalGrammar.Productions.Define(start, new[] { prefix, inlinedNonTerm, suffix }); prod.Actions.Add(new ForeignAction(0, 3)); }
public GrammarAnalysis(Grammar grammar) { this.grammar = grammar; this.tables = new NullableFirstTables(grammar); }
public void Test() { var lParen = new Symbol("("); var rParen = new Symbol(")"); var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var grammar = new Grammar { Symbols = { lParen, rParen, num, ident, qStr }, Conditions = { new Condition("main") { Matchers = { new Matcher( ScanPattern.CreateRegular( null, @"[ \t]+")) { Joint = { new CilMatcher(typeof(void)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"), num) { Joint = { new CilMatcher(typeof(Num)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"), ident) { Joint = { new CilMatcher(typeof(string)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[""](?: \\[""] | [^""])* [""]"), qStr) { Joint = { new CilMatcher(typeof(QStr)) } }, new Matcher( ScanPattern.CreateLiteral("("), lParen), new Matcher( ScanPattern.CreateLiteral(")"), rParen), } } } }; var target = new BootstrapScanner( " (1 (\"bar\" +))", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance), null, ExceptionLogging.Instance); var collector = new Collector<Msg>(); target.Accept(collector); Assert.AreEqual( new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null }, collector.Select(msg => msg.Value).ToArray()); }
private static SymbolBase GetMatcherOutcomeSymbol( Grammar grammar, ICilSymbolResolver symbolResolver, CilSymbolRef mainOutcome, IEnumerable<CilSymbolRef> allOutcomes) { Symbol main = symbolResolver.GetSymbol(mainOutcome); Symbol[] all = (from outcome in allOutcomes select symbolResolver.GetSymbol(outcome)) .ToArray(); switch (all.Length) { case 0: return null; case 1: return main; default: return new AmbiguousSymbol(main, all); } }
private static Condition ConditionFromType(Grammar grammar, Type type) { if (type == null) { return null; } foreach (var cond in grammar.Conditions) { var binding = cond.Joint.The<CilCondition>(); if (binding.ConditionType == type) { return cond; } } throw new InvalidOperationException("Undefined condition: " + type.FullName); }
public TokenIdentitiesSerializer(Grammar grammar) { this.grammar = grammar; }
public void Write(Grammar grammar, TextWriter output) { WriteIndented(grammar, new IndentedTextWriter(output, Indent)); }
private static void InitContextProvider( Grammar grammar, CilContextProvider cilProvider, ForeignContextProvider provider) { provider.Joint.Add(cilProvider); foreach (var cilContext in cilProvider.Contexts) { ForeignContext context; if (grammar.Contexts.FindOrAdd(cilContext.UniqueName, out context)) { context.Joint.Add(cilContext); } provider.Add(context); } }
public void Test() { var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var begin = new Symbol("begin"); var end = new Symbol("end"); var assign = new Symbol(":="); var grammar = new Grammar { Symbols = { num, ident, qStr, begin, end, assign }, Conditions = { new Condition("main") { Matchers = { new Matcher( @"digit+ ('.' digit+)? | '.' digit+", num), new Matcher( @"[01234567]+ 'Q'", num), new Matcher( @"alpha alnum*", ident), new Matcher( @" quot ~(quot | esc)* (esc . ~(quot | esc)* )* quot ", qStr), new Matcher( ScanPattern.CreateLiteral("begin"), begin), new Matcher( ScanPattern.CreateLiteral("end"), end), new Matcher( ScanPattern.CreateLiteral(":="), assign), new Matcher( @"blank+"), } } } }; var target = new TdfaSimulationLexer( "b:=10Q \"foo\"", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance)); var collector = new Collector<Msg>(); target.Accept(collector); Assert.AreEqual( new int[] { ident.Index, assign.Index, num.Index, qStr.Index }, collector.Select(msg => msg.Id).ToArray()); Assert.AreEqual( new object[] { "b", ":=", "10Q", "\"foo\"" }, collector.Select(msg => msg.Value).ToArray()); }
private static Grammar BuildGrammar(CilGrammar definition) { var result = new Grammar(); InitContextProvider( result, definition.GlobalContextProvider, result.GlobalContextProvider); var symbolResolver = definition.SymbolResolver; // Define grammar tokens foreach (var cilSymbol in symbolResolver.Definitions) { Symbol symbol; if (cilSymbol.Type == typeof(Exception)) { cilSymbol.Symbol = symbol = (Symbol)result.Symbols[PredefinedTokens.Error]; symbol.Joint.Add( new CilSymbol { Type = typeof(Exception), Symbol = symbol, Categories = SymbolCategory.DoNotDelete | SymbolCategory.DoNotInsert }); } else { symbol = new Symbol(cilSymbol.Name) { Categories = cilSymbol.Categories, Joint = { cilSymbol } }; result.Symbols.Add(symbol); cilSymbol.Symbol = symbol; } } foreach (CilSymbolFeature<Precedence> feature in definition.Precedence) { var symbol = symbolResolver.GetSymbol(feature.SymbolRef); symbol.Precedence = feature.Value; } foreach (CilSymbolFeature<CilContextProvider> feature in definition.LocalContextProviders) { var symbol = symbolResolver.GetSymbol(feature.SymbolRef); if (symbol != null) { InitContextProvider(result, feature.Value, symbol.LocalContextProvider); } } result.Start = symbolResolver.GetSymbol(definition.Start); // Define grammar rules foreach (var cilProduction in definition.Productions) { Symbol outcome = symbolResolver.GetSymbol(cilProduction.Outcome); var pattern = Array.ConvertAll(cilProduction.Pattern, symbolResolver.GetSymbol); // Try to find existing rules whith same token-signature Production production; if (result.Productions.FindOrAdd(outcome, pattern, out production)) { ForeignContextRef contextRef = CreateActionContextRef(cilProduction.Context); production.Actions.Add(new ForeignAction(pattern.Length, contextRef)); } var action = production.Actions[0]; action.Joint.Add(cilProduction); production.ExplicitPrecedence = cilProduction.Precedence; } // Create conditions to allow referencing them from matchers foreach (CilCondition cilCondition in definition.Conditions) { var cond = CreateCondtion(result, cilCondition); result.Conditions.Add(cond); } // Create matchers foreach (CilCondition cilCondition in definition.Conditions) { var condition = ConditionFromType(result, cilCondition.ConditionType); foreach (var cilMatcher in cilCondition.Matchers) { SymbolBase outcome = GetMatcherOutcomeSymbol(result, symbolResolver, cilMatcher.MainOutcome, cilMatcher.AllOutcomes); var matcher = new Matcher( cilMatcher.Pattern, outcome, #if false context: CreateActionContextRef(cilMatcher.Context), #endif nextCondition: ConditionFromType(result, cilMatcher.NextConditionType), disambiguation: cilMatcher.Disambiguation); matcher.Joint.Add(cilMatcher); condition.Matchers.Add(matcher); } } foreach (var cilMerger in definition.Mergers) { var symbol = symbolResolver.GetSymbol(cilMerger.Symbol); var merger = new Merger(symbol) { Joint = { cilMerger } }; result.Mergers.Add(merger); } foreach (var report in definition.Reports) { result.Reports.Add(report); } return result; }
private static Condition CreateCondtion(Grammar grammar, CilCondition cilCondition) { var result = new Condition(cilCondition.ConditionType.FullName) { Joint = { cilCondition } }; InitContextProvider(grammar, cilCondition.ContextProvider, result.ContextProvider); return result; }
private void WriteIndented(Grammar grammar, IndentedTextWriter output) { WriteSummaryComment(grammar, output); WriteProductions(grammar, output); WriteMatchers(grammar, output); }