public void TestTokenizingComment()
{
const string data = "/* test */";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CommentDelimiters = new[] { new CommentDelimiters("/*", "*/") }
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingStringWithUnexpectedCharacters()
{
const string data = "@";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Assert.That(() => { Token<TokenType> nextToken = tokenizer.GetNextToken(); },
Throws.TypeOf(typeof(UnexpectedCharEncounteredException)));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingEmptyStream()
{
const string data = "";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingNumericLiterals()
{
const string data = "1 123 0x123 0 0x0 -9 5.0 0.0 5.1 -3.5 0xABC 4.5e8 -4.5e8 0x0L 0xABCL 0x123L"
+ " 0x0UL 0xABCUL 0x123UL 12e4L 0. .0 0x0ee 0xFFFFFFFFULL 1.20f";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("1"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("123"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(3));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x123"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(7));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(13));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x0"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(15));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("-9"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(19));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("5.0"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(22));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0.0"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(26));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("5.1"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(30));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("-3.5"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(34));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0xABC"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(39));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("4.5e8"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(45));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("-4.5e8"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(51));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x0L"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(58));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0xABCL"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(63));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x123L"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(70));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x0UL"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(77));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0xABCUL"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(83));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x123UL"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(91));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("12e4L"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(99));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0."));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(105));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo(".0"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(108));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0x0ee"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(111));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("0xFFFFFFFFULL"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(117));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("1.20f"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(131));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
private static Token<TokenType> TestTokenizingNumericLiteral(string data)
{
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
return tokenizer.GetNextToken();
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingPunctuators()
{
const string data = ", ;";
string punctuators = ",;";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
Punctuators = punctuators
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.AreEqual(",", nextToken.Value);
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo(";"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(3));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingSymbolsNotIgnoringWhitespace()
{
const string data = "\t\t\t symbol1 \t \t \t \nsymbol2 ";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Token<TokenType> nextToken = tokenizer.GetNextToken(false);
Assert.That(nextToken.Value, Is.EqualTo("\t\t\t "));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.WHITESPACE));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken(false);
Assert.That(nextToken.Value, Is.EqualTo("symbol1"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(14));
nextToken = tokenizer.GetNextToken(false);
Assert.That(nextToken.Value, Is.EqualTo(" \t \t \t \n"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.WHITESPACE));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(21));
nextToken = tokenizer.GetNextToken(false);
Assert.That(nextToken.Value, Is.EqualTo("symbol2"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(2));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken(false);
Assert.That(nextToken.Value, Is.EqualTo(" "));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.WHITESPACE));
Assert.That(nextToken.LineNumber, Is.EqualTo(2));
Assert.That(nextToken.PositionInLine, Is.EqualTo(8));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestFailingAtTokenizingSymbolsTerminatedByNonPunctuatorOrWhitespace()
{
const string data = "symbol'";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Assert.That(() => { Token<TokenType> nextToken = tokenizer.GetNextToken(); },
Throws.TypeOf(typeof(UnexpectedCharEncounteredException)));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingUnterminatedStringLiteral()
{
const string data = @"""str1";
const char stringLiteralPunctuator = '"';
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
StringLiteralPunctuator = stringLiteralPunctuator
};
Assert.That(() => { Token<TokenType> nextToken = tokenizer.GetNextToken(); },
Throws.TypeOf(typeof(UnexpectedCharEncounteredException)));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingEndCommentAsPunctuators()
{
const string data = "test */";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CommentDelimiters = new[] { new CommentDelimiters("/*", "*/") },
Punctuators = "/*"
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("test"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("*"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(6));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("/"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(7));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestFailingTokenizingUnterminatedComment()
{
const string data = "/* test";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CommentDelimiters = new[] { new CommentDelimiters("/*", "*/") }
};
Assert.That(() => { Token<TokenType> nextToken = tokenizer.GetNextToken(); },
Throws.TypeOf(typeof(UnexpectedCharEncounteredException)));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestFailingTokenizingEndComment()
{
const string data = "test */";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CommentDelimiters = new[] { new CommentDelimiters("/*", "*/") }
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("test"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
Assert.That(() => nextToken = tokenizer.GetNextToken(), Throws.TypeOf(typeof(UnexpectedCharEncounteredException)));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestHasMoreCharactersReturnsTrueWhenStreamNotEmpty()
{
const string data = "a";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Assert.That(tokenizer.HasMoreTokens, Is.True);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestSkippingToNextLineOfTokensWhenTokenTerminatedByNewLine()
{
const string data = " symbol1\nsymbol2";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("symbol1"));
tokenizer.SkipToNextLine();
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("symbol2"));
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingSingleLineCommentWithStuffBeforeAndAfter()
{
const string data = @"a // test
b";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CommentDelimiters = new[] { new CommentDelimiters("//", EndCommentDelimiter.EndOfLine) }
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("a"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("b"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(2));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingStringLiteralWithEscapeCharacter()
{
const string data = @"""str\""test"" ""\\""";
const char stringLiteralPunctuator = '"';
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
StringLiteralPunctuator = stringLiteralPunctuator,
StringEscapeCharacter = '\\'
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("\"str\\\"test\""));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.STRING_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("\"\\\\\""));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.STRING_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(13));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingSymbols()
{
const string data = "symbol1 symbol2";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner);
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("symbol1"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("symbol2"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(9));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingCharLiteralWithEscapeCharacter()
{
const string data = @"'\''";
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
CharLiteralPunctuator = '\'',
StringEscapeCharacter = '\\'
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("'\\''"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.CHAR_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
public void TestTokenizingTokensTerminatedByPunctuators()
{
const string data = "keyword(symbol1, 123, 'str')";
string punctuators = ",()";
const char stringLiteralPunctuator = '\'';
MemoryStream stream = null;
try
{
stream = new MemoryStream(Encoding.ASCII.GetBytes(data));
Scanner scanner = new Scanner(stream);
stream = null;
Tokenizer tokenizer = new Tokenizer(scanner)
{
Punctuators = punctuators,
StringLiteralPunctuator = stringLiteralPunctuator
};
Token<TokenType> nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("keyword"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(1));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("("));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(8));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("symbol1"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.SYMBOL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(9));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo(","));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(16));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("123"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.NUMERIC_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(18));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo(","));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(21));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo("'str'"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.STRING_LITERAL));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(23));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken.Value, Is.EqualTo(")"));
Assert.That(nextToken.Type, Is.EqualTo(TokenType.PUNCTUATOR));
Assert.That(nextToken.LineNumber, Is.EqualTo(1));
Assert.That(nextToken.PositionInLine, Is.EqualTo(28));
nextToken = tokenizer.GetNextToken();
Assert.That(nextToken, Is.Null);
}
finally
{
if (stream != null)
stream.Dispose();
}
}
