public void TestStringEscapedConstantTooShort()
{
var code = "\"\\";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <StringConstantException>(() => tokenizer.EnumerateTokens(code.AsSpan()));
}
public void TestCharConstantEmptyNotLastInFile()
{
var code = @"
'';";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <CharacterConstantException>(() => tokenizer.EnumerateTokens(code.AsSpan()));
}
public void TestInvalidCharacterToken()
{
var code = @"
abc@";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <InvalidTokenParsingException>(() => tokenizer.EnumerateTokens(code.AsSpan()));
}
public void TestCharConstantBadEscape3()
{
var code = @"
'\ab";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <CharacterConstantException>(() => tokenizer.EnumerateTokens(code.AsSpan()));
}
public void TestCharConstantBadEscape2()
{
var code = @"
'\;'";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <UnrecognizedEscapeException>(() => tokenizer.EnumerateTokens(code.AsSpan()));
}
public void TestTokenizerBeginsWithWhitespace()
{
var code = @"
;";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
Assert.IsType <SemiColonToken>(tokens[0]);
}
public void TestTokenizerEndsWithIdentifier()
{
var code = @"
abc";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
Assert.IsType <IdentifierToken>(tokens[0]);
Assert.Equal("abc", ((IdentifierToken)tokens[0]).Name);
}
public void TestEscapedString(string input, string result)
{
var code = $@"
""{input}""";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
var charConstToken = Assert.IsType <StringConstantToken>(tokens[0]);
Assert.Equal(result, charConstToken.Value);
}
public void TestStringConstantEmptyWorks()
{
var code = @"
""""";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
var charConstToken = Assert.IsType <StringConstantToken>(tokens[0]);
Assert.Equal("", charConstToken.Value);
}
public void TestTokenizerBeginsAndEndsWithWhitespace()
{
var code = @"
; abc ";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(2, tokens.Length);
Assert.IsType <SemiColonToken>(tokens[0]);
Assert.IsType <IdentifierToken>(tokens[1]);
Assert.Equal("abc", ((IdentifierToken)tokens[1]).Name);
}
public void VerifyAllSingleCharTokensParseCorrectly()
{
var tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var singleCharTokenClasses = typeof(ISingleCharToken).Assembly.GetTypes().Where(x => !x.IsInterface).Where(x => x.GetInterfaces().Contains(typeof(ISingleCharToken)));
foreach (var tokenClass in singleCharTokenClasses)
{
var tokenCharValueField = tokenClass.GetField("CharValue", BindingFlags.Public | BindingFlags.Static);
char c = (char)tokenCharValueField.GetValue(null);
Assert.IsType(tokenClass, SimpleTokenizer.ParseCharacterToken(c, null));
}
}
public void VerifyAllKeywordTokensParseCorrectly()
{
var tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokenClasses = typeof(IKeywordToken).Assembly.GetTypes().Where(x => x.GetInterfaces().Contains(typeof(IKeywordToken)));
foreach (var tokenClass in tokenClasses)
{
var tokenCharValueField = tokenClass.GetField("KeywordValue", BindingFlags.Public | BindingFlags.Static);
string c = (string)tokenCharValueField.GetValue(null);
//Assert.IsType(tokenClass, SimpleTokenizer.ParseToken(c.AsSpan(), false));
}
}
public void TestPrimaryEscapeSequencesWork(string input, char output)
{
var code = $@"
'{input}'
";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
var charConstToken = Assert.IsType <CharacterConstantToken>(tokens[0]);
Assert.Equal(output, charConstToken.Value);
}
public void TestCharConstantWorksAsOnlyCode()
{
var code = @"
'a'";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(1, tokens.Length);
var charConstToken = Assert.IsType <CharacterConstantToken>(tokens[0]);
Assert.Equal('a', charConstToken.Value);
}
public void TestCharConstantWorksAfterSingleCharTokenWithWhitespace()
{
var code = @"
= 'a'";
ITokenizer tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
var tokens = tokenizer.EnumerateTokens(code.AsSpan());
Assert.Equal(2, tokens.Length);
Assert.IsType <EqualsToken>(tokens[0]);
var charConstToken = Assert.IsType <CharacterConstantToken>(tokens[1]);
Assert.Equal('a', charConstToken.Value);
}
public void TestTokenizerInvalidSingleCharTokenFails()
{
var tokenizer = new Compiler.Tokenizer.SimpleTokenizer();
Assert.Throws <InvalidTokenParsingException>(() => SimpleTokenizer.ParseCharacterToken('c', null));
}