public void TokenizerCanTokenize()
{
_codeReader = new StringReader("console.log('test');");
Tokenizer tokenizer = new Tokenizer();
tokenizer.Initialize(new SourceUnit(_languageContext, _textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
var token = tokenizer.GetNextToken();
token.Should().Equal(new NameToken("console"));
}
public void BasicStructure()
{
_codeReader = new StringReader("console.log('test');");
Tokenizer tokenizer = new Tokenizer();
tokenizer.Initialize(new SourceUnit(_languageContext, _textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { new NameToken("console"), Token.Dot, new NameToken("log"), Token.LeftParenthesisToken, new ConstantValueToken("test"),
Token.RightParenthesisToken, Token.SemicolonToken, Token.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
public void BasicClass()
{
_codeReader = new StringReader("class Test { _i: Number; new(i) { _i = i; } getI() { return _i; } setI(i: Number) { _i = i; } I { get { return _i; } set { _i = value; } } }");
Tokenizer tokenizer = new Tokenizer();
tokenizer.Initialize(new SourceUnit(_languageContext, _textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { Token.ClassToken, new NameToken("Test"), Token.LeftBraceToken, new NameToken("_i"), Token.ColonToken, new NameToken("Number"),
Token.SemicolonToken, Token.NewToken, Token.LeftParenthesisToken, new NameToken("i"), Token.RightParenthesisToken, Token.LeftBraceToken, new NameToken("_i"),
Token.AssignToken, new NameToken("i"), Token.SemicolonToken, Token.RightBraceToken, new NameToken("getI"), Token.LeftParenthesisToken, Token.RightParenthesisToken,
Token.LeftBraceToken, Token.ReturnToken, new NameToken("_i"), Token.SemicolonToken, Token.RightBraceToken, new NameToken("setI"), Token.LeftParenthesisToken, new NameToken("i"),
Token.ColonToken, new NameToken("Number"), Token.RightParenthesisToken, Token.LeftBraceToken, new NameToken("_i"), Token.AssignToken, new NameToken("i"), Token.SemicolonToken,
Token.RightBraceToken, new NameToken("I"), Token.LeftBraceToken, new NameToken("get"), Token.LeftBraceToken, Token.ReturnToken, new NameToken("_i"), Token.SemicolonToken,
Token.RightBraceToken, new NameToken("set"), Token.LeftBraceToken, new NameToken("_i"), Token.AssignToken, new NameToken("value"), Token.SemicolonToken, Token.RightBraceToken,
Token.RightBraceToken, Token.RightBraceToken })
token.Should().Equal(tokenizer.GetNextToken());
}
public void BasicProgram()
{
_codeReader = new StringReader("var a, b;\r\nfunc foo() { console.log('in foo.'); }\nif(true) foo(); for(var i = 0; i < 10; i++)\t log(i); do {/*comment*/}// more comment\n while(false); var x = true ? false : null;");
Tokenizer tokenizer = new Tokenizer();
tokenizer.Initialize(new SourceUnit(_languageContext, _textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { Token.VarToken, new NameToken("a"), Token.CommaToken, new NameToken("b"), Token.SemicolonToken,
Token.FuncToken, new NameToken("foo"), Token.LeftParenthesisToken, Token.RightParenthesisToken,
Token.LeftBraceToken, new NameToken("console"), Token.Dot, new NameToken("log"),
Token.LeftParenthesisToken, new ConstantValueToken("in foo."), Token.RightParenthesisToken,
Token.SemicolonToken, Token.RightBraceToken, Token.IfToken,
Token.LeftParenthesisToken, new ConstantValueToken(true), Token.RightParenthesisToken,
new NameToken("foo"), Token.LeftParenthesisToken, Token.RightParenthesisToken, Token.SemicolonToken,
Token.ForToken, Token.LeftParenthesisToken, Token.VarToken, new NameToken("i"), Token.AssignToken,
new ConstantValueToken(0), Token.SemicolonToken, new NameToken("i"),
Token.LessThanToken, new ConstantValueToken(10), Token.SemicolonToken, new NameToken("i"), Token.IncrementToken, Token.RightParenthesisToken,
new NameToken("log"), Token.LeftParenthesisToken, new NameToken("i"), Token.RightParenthesisToken, Token.SemicolonToken,
Token.DoToken, Token.LeftBraceToken, Token.RightBraceToken, Token.WhileToken, Token.LeftParenthesisToken, new ConstantValueToken(false),
Token.RightParenthesisToken, Token.SemicolonToken,
Token.VarToken, new NameToken("x"), Token.AssignToken, new ConstantValueToken(true), Token.QuestionmarkToken, new ConstantValueToken(false), Token.ColonToken, new ConstantValueToken(null), Token.SemicolonToken,
Token.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
