private static void AddExponentExtension(DoubleExponent token, LexemeAttribute lexem, GenericLexer <DoubleExponent> lexer)
{
if (token == DoubleExponent.DOUBLE_EXP)
{
// callback on end_exponent node
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
string[] items = match.Result.Value.Split(new[] { 'e', 'E' });
double radix = 0;
double.TryParse(items[0].Replace(".", ","), out radix);
double exponent = 0;
double.TryParse(items[1], out exponent);
double value = Math.Pow(radix, exponent);
match.Result.SpanValue = value.ToString().AsMemory();
match.Properties[GenericLexer <DoubleExponent> .DerivedToken] = DoubleExponent.DOUBLE_EXP;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <DoubleExponent> .in_double) // start a in_double node
.Transition(new char[] { 'E', 'e' }) // add a transition on '.' with precondition
.Transition(new char[] { '+', '-' })
.Mark("start_exponent_val")
.RangeTransitionTo('0', '9', "start_exponent_val") // first year digit
.Mark("end_exponent")
.End(GenericToken.Extension) // mark as ending node
.CallBack(callback); // set the ending callback
}
}
public static void SymbolExtension(LispLexer token, LexemeAttribute lexem, GenericLexer <LispLexer> lexer)
{
if (token == LispLexer.SYMBOL)
{
// callback on end_date node
NodeCallback <GenericToken> callback = (FSMMatch <GenericToken> match) =>
{
// this store the token id the the FSMMatch object to be later returned by GenericLexer.Tokenize
match.Properties[GenericLexer <LispLexer> .DerivedToken] = LispLexer.SYMBOL;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
var symbolCharExclusions = new char[]
{ '(', ')', '|', '#', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '.', ' ', '\'', '\r', '\n' };
fsmBuilder.GoTo(GenericLexer <LispLexer> .start)
.ExceptTransition(symbolCharExclusions)
.Mark("end_symbol")
.ExceptTransitionTo(symbolCharExclusions, "end_symbol")
.End(GenericToken.Extension)
.CallBack(callback);
var graph = fsmBuilder.Fsm.ToGraphViz();
;
}
}
///////////////////////////////////////////////////////////////////////
public static Lexeme GetLexeme(
MemberInfo memberInfo
)
{
if (memberInfo != null)
{
try
{
if (memberInfo.IsDefined(
typeof(LexemeAttribute), false))
{
LexemeAttribute flags =
(LexemeAttribute)
memberInfo.GetCustomAttributes(
typeof(LexemeAttribute), false)[0];
return(flags.Lexeme);
}
}
catch
{
// do nothing.
}
}
return(Lexeme.Unknown);
}
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer <Extensions> lexer)
{
if (token == Extensions.DATE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.DATE;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .in_double)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4, "[0-9]")
.End(GenericToken.Extension)
.CallBack(callback);
}
}
public static void AddExtensions(Issue210Token token, LexemeAttribute lexem, GenericLexer <Issue210Token> lexer)
{
if (token == Issue210Token.SPECIAL || token == Issue210Token.QMARK)
{
FSMMatch <GenericToken> Callback(FSMMatch <GenericToken> match)
{
var result = match.Result.Value;
if (result.Length >= 2)
{
if (match.Result.Value[0] == '?' && match.Result.Value.Last() == '?')
{
var section = result.Substring(1, result.Length - 2);
match.Result.SpanValue = section.AsMemory();
match.Properties[GenericLexer <Issue210Token> .DerivedToken] = Issue210Token.SPECIAL;
return(match);
}
Console.WriteLine($"bad lexing {match.Result.Value}");
match.Result.SpanValue = null;
match.Properties[GenericLexer <Issue210Token> .DerivedToken] = default(Issue210Token);
return(match);
}
return(match);
}
lexer.FSMBuilder.GoTo(GenericLexer <Issue210Token> .start)
.Transition('?')
.Mark("qmark")
.ExceptTransition(new[] { '?' }) // moving to first char of a special
.Mark("in_qmark") // now we are really in a potential SPECIAL
.ExceptTransitionTo(new[] { '?' }, "in_qmark")
.Transition('?') // ending ? of a SPECIAL
.End(GenericToken.Extension) // we re done with a SPECIAL
.Mark("end_qmark")
.CallBack(Callback)
.GoTo("qmark")
.TransitionTo('?', "end_qmark");
}
}
public static void AddExtension(Extensions token, LexemeAttribute lexem, GenericLexer <Extensions> lexer)
{
if (token == Extensions.DATE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.DATE;
return(match);
};
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .in_double)
.Transition('.', CheckDate)
.Mark("start_date")
.RepetitionTransition(4, "[0-9]")
.End(GenericToken.Extension)
.CallBack(callback);
}
else if (token == Extensions.CHAINE)
{
NodeCallback <GenericToken> callback = match =>
{
match.Properties[GenericLexer <Extensions> .DerivedToken] = Extensions.CHAINE;
return(match);
};
var quote = '\'';
NodeAction collapseDelimiter = value =>
{
if (value.EndsWith("" + quote + quote))
{
return(value.Substring(0, value.Length - 2) + quote);
}
return(value);
};
var exceptQuote = new[] { quote };
var in_string = "in_string_same";
var escaped = "escaped_same";
var delim = "delim_same";
var fsmBuilder = lexer.FSMBuilder;
fsmBuilder.GoTo(GenericLexer <Extensions> .start)
.Transition(quote)
.Mark(in_string)
.ExceptTransitionTo(exceptQuote, in_string)
.Transition(quote)
.Mark(escaped)
.End(GenericToken.String)
.CallBack(callback)
.Transition(quote)
.Mark(delim)
.Action(collapseDelimiter)
.ExceptTransitionTo(exceptQuote, in_string);
fsmBuilder.GoTo(delim)
.TransitionTo(quote, escaped)
.ExceptTransitionTo(exceptQuote, in_string);
}
}