// Define the scanner with a TokenAutomaton (combined DFA of token types) and // optional special token types to recognize as block comment start and end points // so that nested comments can be handled public Scanner(TokenAutomaton automaton, TokenType blockCommentStart = null, TokenType blockCommentEnd = null) { this.automaton = automaton; this.blockCommentStart = blockCommentStart; this.blockCommentEnd = blockCommentEnd; }
// Define the scanner with a TokenAutomaton (combined DFA of token types) and // optional special token types to recognize as block comment start and end points // so that nested comments can be handled public Scanner(TokenAutomaton automaton, TokenType blockCommentStart = null, TokenType blockCommentEnd = null) { this.automaton = automaton; this.blockCommentStart = blockCommentStart; this.blockCommentEnd = blockCommentEnd; }
private List<Token> GetTokens(TokenAutomaton automaton, string text) { Scanner sc = new Scanner(automaton); IEnumerable<Token> tokens = sc.Tokenize(text, yieldEOF:false); return new List<Token>(tokens); }