public static TokenRecognizer ExactMatchRecognizer(TokenType tokenType, string target)
{
// example: keyword int
// i n t
// (1) ---> (2) ---> (3) ---> ((4))
// 3 transitions, 4 states
var recognizer = new TokenRecognizer(tokenType, new State(1));
for (int i = 1; i <= target.Length; i++)
{
var from = new State(i, StateType.Nonfinite);
var to = new State(i + 1, i == target.Length ? StateType.Finite : StateType.Nonfinite);
var transition = new Transition(from, to, CharTests.Is(target[i - 1]));
recognizer.AddTransition(transition);
}
return(recognizer);
}
static MyLanguage()
{
#region IdentifiersRecognizer [digit | letter| _]{1,}
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Finite);
// set low priority to avoid conflicts with integer constants
IdentifiersRecognizer = new TokenRecognizer(TokenType.Identifier, s1, -1);
IdentifiersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsLetter.Or(CharTests.IsDigit).Or('_')));
IdentifiersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsLetter.Or(CharTests.IsDigit).Or('_')));
}
#endregion
#region DelimetersRecognizer [delim]{1,}
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Finite);
DelimetersRecognizer = new TokenRecognizer(TokenType.Delimiter, s1);
DelimetersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDelimeter));
DelimetersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDelimeter));
}
#endregion
#region StringsRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Finite); // accept unclosed literals
State s3 = new State(3, StateType.Finite);
StringsRecognizer = new TokenRecognizer(TokenType.StringConstant, s1);
StringsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsStringMarker));
StringsRecognizer.AddTransition(new Transition(s2, s2, CharTests.Not(CharTests.IsStringMarker.Or('\r'))));
StringsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsStringMarker));
}
#endregion
#region CharsRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Finite); // accept unclosed char literals
State s3 = new State(3, StateType.Finite);
CharsRecognizer = new TokenRecognizer(TokenType.CharConstant, s1);
CharsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsCharMarker));
CharsRecognizer.AddTransition(new Transition(s2, s2, CharTests.Not(CharTests.IsCharMarker.Or('\r'))));
CharsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsCharMarker));
}
#endregion
#region IntegersRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Finite);
IntegersRecognizer = new TokenRecognizer(TokenType.IntegerConstant, s1);
IntegersRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDigit));
IntegersRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDigit));
}
#endregion
#region FloatsRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Nonfinite);
State s3 = new State(3, StateType.Nonfinite);
State s4 = new State(4, StateType.Finite);
FloatsRecognizer = new TokenRecognizer(TokenType.FloatConstant, s1);
FloatsRecognizer.AddTransition(new Transition(s1, s2, CharTests.IsDigit));
FloatsRecognizer.AddTransition(new Transition(s2, s2, CharTests.IsDigit));
FloatsRecognizer.AddTransition(new Transition(s2, s3, CharTests.IsFloatNumberDelimiter));
FloatsRecognizer.AddTransition(new Transition(s3, s4, CharTests.IsDigit));
FloatsRecognizer.AddTransition(new Transition(s4, s4, CharTests.IsDigit));
}
#endregion
#region CommentsRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Nonfinite);
State s3 = new State(3, StateType.Finite);
CommentsRecognizer = new TokenRecognizer(TokenType.Commentary, s1);
CommentsRecognizer.AddTransition(new Transition(s1, s2, CharTests.Is('/')));
CommentsRecognizer.AddTransition(new Transition(s2, s3, CharTests.Is('/')));
CommentsRecognizer.AddTransition(new Transition(s3, s3, CharTests.Is('\n').Or('\r').Inverse()));
}
#endregion
#region MultiLineCommentsRecognizer
{
State s1 = new State(1, StateType.Nonfinite);
State s2 = new State(2, StateType.Nonfinite);
State s3 = new State(3, StateType.Nonfinite);
State s4 = new State(4, StateType.Nonfinite);
State s5 = new State(5, StateType.Finite);
MultiLineCommentsRecognizer = new TokenRecognizer(TokenType.Commentary, s1);
MultiLineCommentsRecognizer.AddTransition(new Transition(s1, s2, CharTests.Is('/')));
MultiLineCommentsRecognizer.AddTransition(new Transition(s2, s3, CharTests.Is('*')));
MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s3, CharTests.Is('*').Inverse()));
MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s4, CharTests.Is('*')));
MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s3, CharTests.Is('/').Or('*').Inverse()));
MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s4, CharTests.Is('*')));
MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s5, CharTests.Is('/')));
// unclosed multiline comment
MultiLineCommentsRecognizer.AddTransition(new Transition(s3, s5, CharTests.IsEndOfText));
MultiLineCommentsRecognizer.AddTransition(new Transition(s4, s5, CharTests.IsEndOfText));
}
#endregion
#region OperatorRecognizers
OperatorRecognizers = TokenRecognizer.GenerateExactMatchRecognizers(TokenType.Operator, new[]
{
"+",
"-",
"*",
"/",
"=",
"==",
"!=",
">",
"<",
"<=",
">=",
"(",
")",
"{",
"}",
";",
","
});
#endregion
#region Keywords
Keywords = new List <string>(new[]
{
"int",
"float",
"void",
"if",
"else",
"for"
});
#endregion
#region Functions
Functions = new List <string>(new[]
{
"sin",
"cos",
"tan",
"exp",
"read",
"write",
"ln",
"abs"
});
#endregion
}