public void Scanner_BlockCommentTest()
{
Regex a = Regex.Char('a').Star();
Regex b = Regex.Char('b').Star();
// block comment regex /\*([^*]|(\*+[^*/]))\*+/
Regex blockComment = Regex.Concat("/*")
.Concat(Regex.Not('*')
.Union(Regex.Char('*').Plus()
.Concat(Regex.Not('*','/')))
.Star())
.Concat(Regex.Char('*').Plus().Concat(Regex.Char('/')));
Regex whitespace = Regex.Union(" \t\n").Star();
TokenType ttWhitespace = new TokenType("Whitespace", whitespace, priority: TokenType.Priority.Whitespace);
TokenType ttA = new TokenType("a*", a);
TokenType ttB = new TokenType("b*", b);
TokenType ttBlockComment = new TokenType("line comment", blockComment, priority: TokenType.Priority.Whitespace);
TokenAutomaton automaton = TokenType.CombinedAutomaton(ttA, ttB, ttBlockComment, ttWhitespace);
string text = "/* aaa */ \n" +
"bbb \n" +
"/* aaa */ \n" +
"bbb \n" +
"/* aaa \n" +
" aaa */ \n" +
"bbb \n" +
"/* \n" +
" * aaa \n" +
" */ \n" +
"bbb \n" +
"/*** \n" +
" * aaa \n" +
" ***/ \n" +
"bbb \n" +
"/*/ aaa /*/ \n" +
"/*****/ \n" +
"/*///*/ \n" +
"bbb \n" +
"/*** \n" +
" * aaa \n" +
" */ \n" +
"bbb ";
Scanner sc = new Scanner(automaton);
IEnumerable<Token> tokenEnumerable = sc.Tokenize(text, yieldEOF: false);
List<Token> tokens = new List<Token>(tokenEnumerable);
string[] expectedTokens = { "bbb", "bbb", "bbb", "bbb", "bbb", "bbb", "bbb" };
Assert.AreEqual(expectedTokens.Length, tokens.Count);
for (int i = 0; i < expectedTokens.Length; i++)
{
Assert.AreEqual(expectedTokens[i], tokens[i].Lexeme);
Assert.AreEqual(ttB, tokens[i].Type);
}
}
private List<Token> GetTokens(TokenAutomaton automaton, string text)
{
Scanner sc = new Scanner(automaton);
IEnumerable<Token> tokens = sc.Tokenize(text, yieldEOF:false);
return new List<Token>(tokens);
}
public void Scanner_NestedCommentHackTest3()
{
Regex a = Regex.Char('a').Star();
Regex b = Regex.Char('b').Star();
// hacky solution that allows for nested comments
Regex reBlockCommentStart = Regex.Concat("/*");
Regex reBlockCommentEnd = Regex.Concat("*/");
Regex whitespace = Regex.Union(" \t\n").Star();
TokenType ttWhitespace = new TokenType("Whitespace", whitespace, priority: TokenType.Priority.Whitespace);
TokenType ttA = new TokenType("a*", a);
TokenType ttB = new TokenType("b*", b);
TokenType ttBlockCommentStart = new TokenType("block comment start", reBlockCommentStart);
TokenType ttBlockCommentEnd = new TokenType("block comment end", reBlockCommentEnd);
TokenAutomaton automaton = TokenType.CombinedAutomaton(ttA, ttB, ttBlockCommentStart, ttBlockCommentEnd, ttWhitespace);
string text = "bbb /* aaa";
Scanner sc = new Scanner(automaton, ttBlockCommentStart, ttBlockCommentEnd);
IEnumerable<Token> tokenEnumerable = sc.Tokenize(text, yieldEOF: false);
List<Token> tokens = new List<Token>(tokenEnumerable);
string[] expectedTokens = { "bbb" };
Assert.AreEqual(expectedTokens.Length, tokens.Count);
for (int i = 0; i < expectedTokens.Length; i++)
{
Assert.AreEqual(expectedTokens[i], tokens[i].Lexeme);
Assert.AreEqual(ttB, tokens[i].Type);
}
}