private void InitializeStaticLexer(IdentifierType idType = IdentifierType.Alpha,
params GenericToken[] staticTokens)
{
FSMBuilder = new FSMLexerBuilder <GenericToken>();
StringCounter = 0;
// conf
FSMBuilder.IgnoreWS()
.WhiteSpace(' ')
.WhiteSpace('\t')
.IgnoreEOL();
// start machine definition
FSMBuilder.Mark(start);
if (staticTokens.ToList().Contains(GenericToken.Identifier) ||
staticTokens.ToList().Contains(GenericToken.KeyWord))
{
InitializeIdentifier(idType);
}
//numeric
if (staticTokens.ToList().Contains(GenericToken.Int) || staticTokens.ToList().Contains(GenericToken.Double))
{
FSMBuilder = FSMBuilder.GoTo(start)
.RangeTransition('0', '9')
.Mark(in_int)
.RangeTransitionTo('0', '9', in_int)
.End(GenericToken.Int);
if (staticTokens.ToList().Contains(GenericToken.Double))
{
FSMBuilder.Transition('.')
.Mark(start_double)
.RangeTransition('0', '9')
.Mark(in_double)
.RangeTransitionTo('0', '9', in_double)
.End(GenericToken.Double);
}
}
LexerFsm = FSMBuilder.Fsm;
}
private void InitializeStaticLexer(Config config, GenericToken[] staticTokens)
{
FSMBuilder = new FSMLexerBuilder <GenericToken>();
StringCounter = 0;
// conf
FSMBuilder
.IgnoreWS(config.IgnoreWS)
.WhiteSpace(config.WhiteSpace)
.IgnoreEOL(config.IgnoreEOL);
// start machine definition
FSMBuilder.Mark(start);
if (staticTokens.Contains(GenericToken.Identifier) || staticTokens.Contains(GenericToken.KeyWord))
{
InitializeIdentifier(config);
}
// numeric
if (staticTokens.Contains(GenericToken.Int) || staticTokens.Contains(GenericToken.Double))
{
FSMBuilder = FSMBuilder.GoTo(start)
.RangeTransition('0', '9')
.Mark(in_int)
.RangeTransitionTo('0', '9', in_int)
.End(GenericToken.Int);
if (staticTokens.Contains(GenericToken.Double))
{
FSMBuilder.Transition('.')
.Mark(start_double)
.RangeTransition('0', '9')
.Mark(in_double)
.RangeTransitionTo('0', '9', in_double)
.End(GenericToken.Double);
}
}
LexerFsm = FSMBuilder.Fsm;
}
private static void testLexerBuilder()
{
var builder = new FSMLexerBuilder <JsonToken>();
// conf
builder.IgnoreWS()
.WhiteSpace(' ')
.WhiteSpace('\t')
.IgnoreEOL();
// start machine definition
builder.Mark("start");
// string literal
builder.Transition('\"')
.Mark("in_string")
.ExceptTransitionTo(new[] { '\"', '\\' }, "in_string")
.Transition('\\')
.Mark("escape")
.AnyTransitionTo(' ', "in_string")
.Transition('\"')
.End(JsonToken.STRING)
.Mark("string_end")
.CallBack(match =>
{
match.Result.Value = match.Result.Value.ToUpper();
return(match);
});
// accolades
builder.GoTo("start")
.Transition('{')
.End(JsonToken.ACCG);
builder.GoTo("start")
.Transition('}')
.End(JsonToken.ACCD);
// corchets
builder.GoTo("start")
.Transition('[')
.End(JsonToken.CROG);
builder.GoTo("start")
.Transition(']')
.End(JsonToken.CROD);
// 2 points
builder.GoTo("start")
.Transition(':')
.End(JsonToken.COLON);
// comma
builder.GoTo("start")
.Transition(',')
.End(JsonToken.COMMA);
//numeric
builder.GoTo("start")
.RangeTransition('0', '9')
.Mark("in_int")
.RangeTransitionTo('0', '9', "in_int")
.End(JsonToken.INT)
.Transition('.')
.Mark("start_double")
.RangeTransition('0', '9')
.Mark("in_double")
.RangeTransitionTo('0', '9', "in_double")
.End(JsonToken.DOUBLE);
var code = "{\n\"d\" : 42.42 ,\n\"i\" : 42 ,\n\"s\" : \"quarante-deux\",\n\"s2\":\"a\\\"b\"\n}";
//code = File.ReadAllText("test.json");
var lex = builder.Fsm;
var r = lex.Run(code, 0);
var total = "";
while (r.IsSuccess)
{
var msg = $"{r.Result.TokenID} : {r.Result.Value} @{r.Result.Position}";
total += msg + "\n";
Console.WriteLine(msg);
r = lex.Run(code);
}
}