public void SimpleTest()
{
var text = "var a = 20";
var source = new Shovel.SourceFile () {
FileName = "test.sho",
Content = text
};
var tokenizer = new Shovel.Compiler.Tokenizer (source);
// Expect 5 tokens because a 'filename' token is inserted.
Assert.AreEqual (5, tokenizer.Tokens.Count);
var sb = new StringBuilder ();
sb.AppendLine ();
foreach (var token in tokenizer.Tokens) {
sb.AppendFormat (
"[Token: Type={0}, StartPos={1}, EndPos={2}, Content='{3}']\n",
token.Type, token.StartPos, token.EndPos, token.Content);
}
Assert.AreEqual (@"
[Token: Type=FileName, StartPos=0, EndPos=0, Content='test.sho']
[Token: Type=Keyword, StartPos=0, EndPos=2, Content='var']
[Token: Type=Identifier, StartPos=4, EndPos=4, Content='a']
[Token: Type=Punctuation, StartPos=6, EndPos=6, Content='=']
[Token: Type=Number, StartPos=8, EndPos=9, Content='20']
", sb.ToString().TrimCarriageReturn());
}
public void SimpleTest()
{
var text = "var a = 20";
var source = new Shovel.SourceFile()
{
FileName = "test.sho",
Content = text
};
var tokenizer = new Shovel.Compiler.Tokenizer(source);
// Expect 5 tokens because a 'filename' token is inserted.
Assert.AreEqual(5, tokenizer.Tokens.Count);
var sb = new StringBuilder();
sb.AppendLine();
foreach (var token in tokenizer.Tokens)
{
sb.AppendFormat(
"[Token: Type={0}, StartPos={1}, EndPos={2}, Content='{3}']\n",
token.Type, token.StartPos, token.EndPos, token.Content);
}
Assert.AreEqual(@"
[Token: Type=FileName, StartPos=0, EndPos=0, Content='test.sho']
[Token: Type=Keyword, StartPos=0, EndPos=2, Content='var']
[Token: Type=Identifier, StartPos=4, EndPos=4, Content='a']
[Token: Type=Punctuation, StartPos=6, EndPos=6, Content='=']
[Token: Type=Number, StartPos=8, EndPos=9, Content='20']
", sb.ToString().TrimCarriageReturn());
}
public void TestUnfinishedLiteralString()
{
var text = "var a = '20";
var source = new Shovel.SourceFile () {
FileName = "test.sho",
Content = text
};
Utils.ExpectException<ShovelException> (() => {
Assert.IsNotNull (new Shovel.Compiler.Tokenizer (source).Tokens);
},
(ex) => {
Assert.IsNotNull (ex);
Assert.AreEqual ("Expected an end quote, but reached the end of file.", ex.Message);
Assert.AreEqual ("test.sho", ex.FileName);
Assert.AreEqual (true, ex.AtEof);
}
);
}
public void TestUnfinishedLiteralString()
{
var text = "var a = '20";
var source = new Shovel.SourceFile()
{
FileName = "test.sho",
Content = text
};
Utils.ExpectException <ShovelException> (() => {
Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens);
},
(ex) => {
Assert.IsNotNull(ex);
Assert.AreEqual("Expected an end quote, but reached the end of file.", ex.Message);
Assert.AreEqual("test.sho", ex.FileName);
Assert.AreEqual(true, ex.AtEof);
}
);
}
public void TestUnfinishedMultilineComment()
{
var text = "var a = 10 /* this is a comment";
var source = new Shovel.SourceFile()
{
FileName = "test.sho",
Content = text
};
Utils.ExpectException <ShovelException>(() =>
{
Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens);
},
(ex) =>
{
Assert.IsNotNull(ex);
Assert.AreEqual("Reached the end of the file while parsing a multiline comment.", ex.Message);
Assert.AreEqual("test.sho", ex.FileName);
Assert.AreEqual(true, ex.AtEof);
}
);
}
public void TestUnknownCharacter()
{
var text = "var a = `";
var source = new Shovel.SourceFile()
{
FileName = "test.sho",
Content = text
};
Utils.ExpectException <ShovelException> (() => {
Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens);
},
(ex) => {
Assert.IsNotNull(ex);
Assert.AreEqual(@"Unexpected character '`'.
file 'test.sho' line 1: var a = `
file 'test.sho' line 1: ^", ex.Message);
Assert.AreEqual("test.sho", ex.FileName);
Assert.AreEqual(false, ex.AtEof);
Assert.AreEqual(1, ex.Line);
Assert.AreEqual(9, ex.Column);
}
);
}
public void TestUnfinishedMultilineComment()
{
var text = "var a = 10 /* this is a comment";
var source = new Shovel.SourceFile()
{
FileName = "test.sho",
Content = text
};
Utils.ExpectException<ShovelException>(() =>
{
Assert.IsNotNull(new Shovel.Compiler.Tokenizer(source).Tokens);
},
(ex) =>
{
Assert.IsNotNull(ex);
Assert.AreEqual("Reached the end of the file while parsing a multiline comment.", ex.Message);
Assert.AreEqual("test.sho", ex.FileName);
Assert.AreEqual(true, ex.AtEof);
}
);
}
public void TestUnknownCharacter()
{
var text = "var a = `";
var source = new Shovel.SourceFile () {
FileName = "test.sho",
Content = text
};
Utils.ExpectException<ShovelException> (() => {
Assert.IsNotNull (new Shovel.Compiler.Tokenizer (source).Tokens);
},
(ex) => {
Assert.IsNotNull (ex);
Assert.AreEqual (@"Unexpected character '`'.
file 'test.sho' line 1: var a = `
file 'test.sho' line 1: ^", ex.Message);
Assert.AreEqual ("test.sho", ex.FileName);
Assert.AreEqual (false, ex.AtEof);
Assert.AreEqual (1, ex.Line);
Assert.AreEqual (9, ex.Column);
}
);
}
