public void BasicProgram()
{
codeReader = new StringReader("var a, b;\r\nfunction foo() { console.log('in foo.'); }\nif(true) foo(); for(var i = 0; i < 10; i++)\t log(i); do {/*comment*/}// more comment\n while(false); var x = true ? false : null;");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { Tokens.VarToken, new NameToken("a"), Tokens.CommaToken, new NameToken("b"), Tokens.SemicolonToken,
Tokens.FunctionToken, new NameToken("foo"), Tokens.LeftParenthesisToken, Tokens.RightParenthesisToken,
Tokens.LeftBraceToken, new NameToken("console"), Tokens.DotToken, new NameToken("log"),
Tokens.LeftParenthesisToken, new ConstantValueToken("in foo."), Tokens.RightParenthesisToken,
Tokens.SemicolonToken, Tokens.RightBraceToken, Tokens.IfToken,
Tokens.LeftParenthesisToken, new ConstantValueToken(true), Tokens.RightParenthesisToken,
new NameToken("foo"), Tokens.LeftParenthesisToken, Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.ForToken, Tokens.LeftParenthesisToken, Tokens.VarToken, new NameToken("i"), Tokens.AssignToken,
new ConstantValueToken(0), Tokens.SemicolonToken, new NameToken("i"),
Tokens.LessThanToken, new ConstantValueToken(10), Tokens.SemicolonToken, new NameToken("i"), Tokens.IncrementToken, Tokens.RightParenthesisToken,
new NameToken("log"), Tokens.LeftParenthesisToken, new NameToken("i"), Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.DoToken, Tokens.LeftBraceToken, Tokens.RightBraceToken, Tokens.WhileToken, Tokens.LeftParenthesisToken, new ConstantValueToken(false),
Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.VarToken, new NameToken("x"), Tokens.AssignToken, new ConstantValueToken(true), Tokens.QuestionmarkToken, new ConstantValueToken(false), Tokens.ColonToken, new ConstantValueToken(null), Tokens.SemicolonToken,
Tokens.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
public void CanReset()
{
codeReader = new StringReader("console.log('test');");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { new NameToken("console"), Tokens.DotToken, new NameToken("log"), Tokens.LeftParenthesisToken, new ConstantValueToken("test"),
Tokens.RightParenthesisToken, Tokens.SemicolonToken, Tokens.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
// Second take
codeReader = new StringReader("console.log('test');");
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { new NameToken("console"), Tokens.DotToken, new NameToken("log"), Tokens.LeftParenthesisToken, new ConstantValueToken("test"),
Tokens.RightParenthesisToken, Tokens.SemicolonToken, Tokens.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
public void TokenizerCanTokenize()
{
codeReader = new StringReader("console.log('test');");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
var token = tokenizer.GetNextToken();
token.Should().Equal(new NameToken("console"));
}
