// Define the scanner with a TokenAutomaton (combined DFA of token types) and
// optional special token types to recognize as block comment start and end points
// so that nested comments can be handled
public Scanner(TokenAutomaton automaton, TokenType blockCommentStart = null, TokenType blockCommentEnd = null)
{
this.automaton = automaton;
this.blockCommentStart = blockCommentStart;
this.blockCommentEnd = blockCommentEnd;
}
// Define the scanner with a TokenAutomaton (combined DFA of token types) and
// optional special token types to recognize as block comment start and end points
// so that nested comments can be handled
public Scanner(TokenAutomaton automaton, TokenType blockCommentStart = null, TokenType blockCommentEnd = null)
{
this.automaton = automaton;
this.blockCommentStart = blockCommentStart;
this.blockCommentEnd = blockCommentEnd;
}
private List<Token> GetTokens(TokenAutomaton automaton, string text)
{
Scanner sc = new Scanner(automaton);
IEnumerable<Token> tokens = sc.Tokenize(text, yieldEOF:false);
return new List<Token>(tokens);
}