public void BasicProgram()
{
codeReader = new StringReader("var a, b;\r\nfunction foo() { console.log('in foo.'); }\nif(true) foo(); for(var i = 0; i < 10; i++)\t log(i); do {/*comment*/}// more comment\n while(false); var x = true ? false : null;");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { Tokens.VarToken, new NameToken("a"), Tokens.CommaToken, new NameToken("b"), Tokens.SemicolonToken,
Tokens.FunctionToken, new NameToken("foo"), Tokens.LeftParenthesisToken, Tokens.RightParenthesisToken,
Tokens.LeftBraceToken, new NameToken("console"), Tokens.DotToken, new NameToken("log"),
Tokens.LeftParenthesisToken, new ConstantValueToken("in foo."), Tokens.RightParenthesisToken,
Tokens.SemicolonToken, Tokens.RightBraceToken, Tokens.IfToken,
Tokens.LeftParenthesisToken, new ConstantValueToken(true), Tokens.RightParenthesisToken,
new NameToken("foo"), Tokens.LeftParenthesisToken, Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.ForToken, Tokens.LeftParenthesisToken, Tokens.VarToken, new NameToken("i"), Tokens.AssignToken,
new ConstantValueToken(0), Tokens.SemicolonToken, new NameToken("i"),
Tokens.LessThanToken, new ConstantValueToken(10), Tokens.SemicolonToken, new NameToken("i"), Tokens.IncrementToken, Tokens.RightParenthesisToken,
new NameToken("log"), Tokens.LeftParenthesisToken, new NameToken("i"), Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.DoToken, Tokens.LeftBraceToken, Tokens.RightBraceToken, Tokens.WhileToken, Tokens.LeftParenthesisToken, new ConstantValueToken(false),
Tokens.RightParenthesisToken, Tokens.SemicolonToken,
Tokens.VarToken, new NameToken("x"), Tokens.AssignToken, new ConstantValueToken(true), Tokens.QuestionmarkToken, new ConstantValueToken(false), Tokens.ColonToken, new ConstantValueToken(null), Tokens.SemicolonToken,
Tokens.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
public void BasicStructure()
{
codeReader = new StringReader("console.log('test');");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
foreach (var token in new Token[] { new NameToken("console"), Tokens.DotToken, new NameToken("log"), Tokens.LeftParenthesisToken, new ConstantValueToken("test"),
Tokens.RightParenthesisToken, Tokens.SemicolonToken, Tokens.EndOfFile })
token.Should().Equal(tokenizer.GetNextToken());
}
private TotemParser(CompilerContext context, TotemTokenizer tokenizer, ErrorSink errorSink, ParserSink parserSink)
{
ContractUtils.RequiresNotNull(tokenizer, "tokenizer");
ContractUtils.RequiresNotNull(errorSink, "errorSink");
ContractUtils.RequiresNotNull(parserSink, "parserSink");
tokenizer.ErrorSink = new TokenizerErrorSink(this);
_tokenizer = tokenizer;
_errors = errorSink;
if (parserSink != ParserSink.Null)
_sink = parserSink;
_context = context;
Reset(tokenizer.SourceUnit);
}
private static TotemParser CreateParserWorker(CompilerContext context, TotemLanguageOptions options)
{
ContractUtils.RequiresNotNull(context, "context");
ContractUtils.RequiresNotNull(options, "options");
TotemCompilerOptions compilerOptions = context.Options as TotemCompilerOptions;
if (options == null)
throw new NullReferenceException("Invalid compiler options");
SourceCodeReader reader;
try
{
reader = context.SourceUnit.GetReader();
}
catch (IOException e)
{
context.Errors.Add(context.SourceUnit, e.Message, SourceSpan.Invalid, 0, Severity.Error);
throw;
}
TotemTokenizer tokenizer = new TotemTokenizer(context.Errors, compilerOptions);
tokenizer.Initialize(null, reader, context.SourceUnit, SourceLocation.MinValue);
TotemParser result = new TotemParser(context, tokenizer, context.Errors, context.ParserSink);
result._sourceReader = reader;
return result;
}
public void TokenizerIsInitializeable()
{
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
}
public void TokenizerCanTokenize()
{
codeReader = new StringReader("console.log('test');");
TotemTokenizer tokenizer = new TotemTokenizer();
tokenizer.Initialize(new SourceUnit(languageContext, textContentProvider, "C:\\A\\Fake\\Path.txt", SourceCodeKind.File));
var token = tokenizer.GetNextToken();
token.Should().Equal(new NameToken("console"));
}
