public TdfaSimulationLexer(TextReader textSource, ScannerDescriptor descriptor) { this.text = textSource.ReadToEnd(); this.descriptor = descriptor; Dictionary <string, int> literalToAction = new Dictionary <string, int>(); var root = descriptor.MakeAst(literalToAction); var regTree = new RegularTree(root); var algorithm = new RegularToTdfaAlgorithm(regTree, literalToAction); DescribeTdfa(algorithm.Data); this.tdfa = new TdfaSimulation(algorithm.Data); int count = descriptor.Matchers.Count; this.tokenFactories = new TokenFactoryDelegate[count]; for (int i = 0; i != count; ++i) { if (descriptor.Matchers[i].Outcome != null) { tokenFactories[i] = BuildTokenFactory(descriptor.Matchers[i]); } } }
public void Test() { var lParen = new Symbol("("); var rParen = new Symbol(")"); var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var grammar = new Grammar { Symbols = { lParen, rParen, num, ident, qStr }, Conditions = { new Condition("main") { Matchers = { new Matcher(@"blank+"), new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num), new Matcher( @"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*", ident), new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr), new Matcher(ScanPattern.CreateLiteral("("), lParen), new Matcher(ScanPattern.CreateLiteral(")"), rParen), } } } }; var target = new DfaSimulationLexer( " (1 (\"bar\" +))", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance)); var collector = new Collector <Msg>(); target.Accept(collector); Assert.AreEqual( new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index }, collector.Select(msg => msg.Id).ToArray()); Assert.AreEqual( new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" }, collector.Select(msg => msg.Value).ToArray()); }
public DfaSimulationLexer(TextReader textSource, ScannerDescriptor descriptor) { this.text = textSource.ReadToEnd(); this.descriptor = descriptor; var root = descriptor.MakeAst(); var regTree = new RegularTree(root); var algorithm = new RegularToDfaAlgorithm(regTree); this.dfa = new DfaSimulation(algorithm.Data); int count = descriptor.Matchers.Count; this.tokenFactories = new TokenFactoryDelegate[count]; for (int i = 0; i != count; ++i) { if (descriptor.Matchers[i].Outcome != null) { tokenFactories[i] = BuildTokenFactory(descriptor.Matchers[i]); } } }
public TdfaSimulationLexer(string text, ScannerDescriptor descriptor) : this(new StringReader(text), descriptor) { }
public void Test() { var lParen = new Symbol("("); var rParen = new Symbol(")"); var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var grammar = new Grammar { Symbols = { lParen, rParen, num, ident, qStr }, Conditions = { new Condition("main") { Matchers = { new Matcher( ScanPattern.CreateRegular( null, @"[ \t]+")) { Joint ={ new CilMatcher(typeof(void)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"), num) { Joint ={ new CilMatcher(typeof(Num)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"), ident) { Joint ={ new CilMatcher(typeof(string)) } }, new Matcher( ScanPattern.CreateRegular( null, @"[""](?: \\[""] | [^""])* [""]"), qStr) { Joint ={ new CilMatcher(typeof(QStr)) } }, new Matcher( ScanPattern.CreateLiteral("("), lParen), new Matcher( ScanPattern.CreateLiteral(")"), rParen), } } } }; var target = new BootstrapScanner( " (1 (\"bar\" +))", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance), null, ExceptionLogging.Instance); var collector = new Collector <Msg>(); target.Accept(collector); Assert.AreEqual( new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null }, collector.Select(msg => msg.Value).ToArray()); }
public void Test() { var num = new Symbol("NUM"); var ident = new Symbol("ID"); var qStr = new Symbol("QSTR"); var begin = new Symbol("begin"); var end = new Symbol("end"); var assign = new Symbol(":="); var grammar = new Grammar { Symbols = { num, ident, qStr, begin, end, assign }, Conditions = { new Condition("main") { Matchers = { new Matcher( @"digit+ ('.' digit+)? | '.' digit+", num), new Matcher( @"[01234567]+ 'Q'", num), new Matcher( @"alpha alnum*", ident), new Matcher( @" quot ~(quot | esc)* (esc . ~(quot | esc)* )* quot ", qStr), new Matcher( ScanPattern.CreateLiteral("begin"), begin), new Matcher( ScanPattern.CreateLiteral("end"), end), new Matcher( ScanPattern.CreateLiteral(":="), assign), new Matcher( @"blank+"), } } } }; var target = new TdfaSimulationLexer( "b:=10Q \"foo\"", ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance)); var collector = new Collector<Msg>(); target.Accept(collector); Assert.AreEqual( new int[] { ident.Index, assign.Index, num.Index, qStr.Index }, collector.Select(msg => msg.Id).ToArray()); Assert.AreEqual( new object[] { "b", ":=", "10Q", "\"foo\"" }, collector.Select(msg => msg.Value).ToArray()); }