public TdfaSimulationLexer(TextReader textSource, ScannerDescriptor descriptor)
{
this.text = textSource.ReadToEnd();
this.descriptor = descriptor;
Dictionary <string, int> literalToAction = new Dictionary <string, int>();
var root = descriptor.MakeAst(literalToAction);
var regTree = new RegularTree(root);
var algorithm = new RegularToTdfaAlgorithm(regTree, literalToAction);
DescribeTdfa(algorithm.Data);
this.tdfa = new TdfaSimulation(algorithm.Data);
int count = descriptor.Matchers.Count;
this.tokenFactories = new TokenFactoryDelegate[count];
for (int i = 0; i != count; ++i)
{
if (descriptor.Matchers[i].Outcome != null)
{
tokenFactories[i] = BuildTokenFactory(descriptor.Matchers[i]);
}
}
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(@"blank+"),
new Matcher(@"digit+ ('.' digit+)? | '.' digit+", num),
new Matcher(
@"(alpha | [:.!@#$%^&|?*/+*=\\_-]) (alnum | [:.!@#$%^&|?*/+*=\\_-])*",
ident),
new Matcher("'\"' ('\\\\\"' | ~'\"')* '\"'", qStr),
new Matcher(ScanPattern.CreateLiteral("("), lParen),
new Matcher(ScanPattern.CreateLiteral(")"), rParen),
}
}
}
};
var target = new DfaSimulationLexer(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector <Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { lParen.Index, num.Index, lParen.Index, qStr.Index, ident.Index, rParen.Index, rParen.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "(", "1", "(", "\"bar\"", "+", ")", ")" },
collector.Select(msg => msg.Value).ToArray());
}
public DfaSimulationLexer(TextReader textSource, ScannerDescriptor descriptor)
{
this.text = textSource.ReadToEnd();
this.descriptor = descriptor;
var root = descriptor.MakeAst();
var regTree = new RegularTree(root);
var algorithm = new RegularToDfaAlgorithm(regTree);
this.dfa = new DfaSimulation(algorithm.Data);
int count = descriptor.Matchers.Count;
this.tokenFactories = new TokenFactoryDelegate[count];
for (int i = 0; i != count; ++i)
{
if (descriptor.Matchers[i].Outcome != null)
{
tokenFactories[i] = BuildTokenFactory(descriptor.Matchers[i]);
}
}
}
public TdfaSimulationLexer(string text, ScannerDescriptor descriptor)
: this(new StringReader(text), descriptor)
{
}
public void Test()
{
var lParen = new Symbol("(");
var rParen = new Symbol(")");
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var grammar = new Grammar
{
Symbols =
{
lParen,
rParen,
num,
ident,
qStr
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
ScanPattern.CreateRegular(
null,
@"[ \t]+"))
{
Joint ={ new CilMatcher(typeof(void)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[0-9]+(?:[.][0-9]+)? | [.][0-9]+"),
num)
{
Joint ={ new CilMatcher(typeof(Num)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[a-zA-Z:.!@#$%^&|?*/+*=\\_-][a-zA-Z:\d.!@#$%^&|?*/+*=\\_-]*"),
ident)
{
Joint ={ new CilMatcher(typeof(string)) }
},
new Matcher(
ScanPattern.CreateRegular(
null,
@"[""](?: \\[""] | [^""])* [""]"),
qStr)
{
Joint ={ new CilMatcher(typeof(QStr)) }
},
new Matcher(
ScanPattern.CreateLiteral("("),
lParen),
new Matcher(
ScanPattern.CreateLiteral(")"),
rParen),
}
}
}
};
var target = new BootstrapScanner(
" (1 (\"bar\" +))",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance),
null,
ExceptionLogging.Instance);
var collector = new Collector <Msg>();
target.Accept(collector);
Assert.AreEqual(
new object[] { null, new Num("1"), null, new QStr("bar"), "+", null, null },
collector.Select(msg => msg.Value).ToArray());
}
public void Test()
{
var num = new Symbol("NUM");
var ident = new Symbol("ID");
var qStr = new Symbol("QSTR");
var begin = new Symbol("begin");
var end = new Symbol("end");
var assign = new Symbol(":=");
var grammar = new Grammar
{
Symbols =
{
num,
ident,
qStr,
begin,
end,
assign
},
Conditions =
{
new Condition("main")
{
Matchers =
{
new Matcher(
@"digit+ ('.' digit+)? | '.' digit+",
num),
new Matcher(
@"[01234567]+ 'Q'",
num),
new Matcher(
@"alpha alnum*",
ident),
new Matcher(
@"
quot
~(quot | esc)*
(esc . ~(quot | esc)* )*
quot
",
qStr),
new Matcher(
ScanPattern.CreateLiteral("begin"),
begin),
new Matcher(
ScanPattern.CreateLiteral("end"),
end),
new Matcher(
ScanPattern.CreateLiteral(":="),
assign),
new Matcher(
@"blank+"),
}
}
}
};
var target = new TdfaSimulationLexer(
"b:=10Q \"foo\"",
ScannerDescriptor.FromScanRules(grammar.Conditions[0].Matchers, ExceptionLogging.Instance));
var collector = new Collector<Msg>();
target.Accept(collector);
Assert.AreEqual(
new int[] { ident.Index, assign.Index, num.Index, qStr.Index },
collector.Select(msg => msg.Id).ToArray());
Assert.AreEqual(
new object[] { "b", ":=", "10Q", "\"foo\"" },
collector.Select(msg => msg.Value).ToArray());
}
