static void Run(string code)
{
List <CodeTokenizer.Token> tokens = null;
StatementBlock block = null;
try {
tokens = CodeTokenizer.Tokenize(code, file);
var tokensEnum = (IEnumerator <CodeTokenizer.Token>)tokens.GetEnumerator();
block = TokenParser.Parse(ref tokensEnum, enviroment, CodePosition.GetExternal());
} catch (WordScriptException ex) {
Console.WriteLine(ex.Message);
}
if (tokens != null)
{
foreach (var token in tokens)
{
Console.WriteLine(token.ToString());
}
}
Console.WriteLine("");
if (block != null)
{
foreach (var node in block.GetSyntaxNodes())
{
Console.WriteLine(node.Debug());
}
var ret = block.Evaluate();
Console.WriteLine(ret?.ToString() ?? "null");
}
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
private Local(ILocal originalLocal, StatementBlock homeStatementBlock, int localIndex, LocalBuilder localBuilder)
{
m_OriginalLocal = originalLocal;
m_HomeStatementBlock = homeStatementBlock;
m_LocalIndex = localIndex;
m_LocalBuilder = localBuilder;
}
/// <inheritdoc />
public void FormatToken(IScriptToken token, StringBuilder resulttext, IFormatterCollection formatters, int depth = 0, bool mustindent = false)
{
StatementBlock block = (StatementBlock)token;
if (depth > -1)
{
resulttext.AppendLine(" {");
}
foreach (IScriptToken child in block.Children)
{
formatters[child].FormatToken(child, resulttext, formatters, depth >= 0 ? depth : 0, true);
resulttext.AppendLine();
}
if (depth > -1)
{
for (int i = 0; i < depth - 1; ++i)
{
resulttext.Append('\t');
}
resulttext.Append('}');
}
if (depth < 0)
{
while (resulttext[resulttext.Length - 1] == '\r' || resulttext[resulttext.Length - 1] == '\n')
{
--resulttext.Length;
}
}
}
public void ShouldParseMultipleAssignments()
{
var assignment = new Assignment(new[] { new Variable("a"), new Variable("b") },
new[] { new ConstantExpression(Constants.One) }, true);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("local a,b = 1"));
}
/// <summary>
/// Processes the statement with specified cpu context. Returns false if we couldn't read due to invalid memory address.
/// </summary>
/// <param name="cpu">The cpu.</param>
/// <param name="parent">The parent.</param>
/// <param name="stack">The stack.</param>
/// <returns></returns>
internal override bool Process(CPURegisters cpu, StatementBlock parent, List <IntPtr> stack)
{
if (!this.Operand1.Process(cpu, parent, stack))
{
return(false);
}
IntPtr op1 = stack[stack.Count - 1];
stack.RemoveAt(stack.Count - 1);
if (!this.Operand2.Process(cpu, parent, stack))
{
return(false);
}
IntPtr op2 = stack[stack.Count - 1];
stack.RemoveAt(stack.Count - 1);
ulong op2_t = Main.Is64Bit ? op2.ToUInt64() : op2.ToUInt32();
IntPtr result = this.Operator.Func(op1, op2_t);
stack.Add(result);
return(true);
}
public void ShouldParseSimpleAssignment()
{
var assignment = new Assignment(new[] {new Variable("a")},
new[] {new ConstantExpression(Constants.One)}, false);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("a = 1"));
}
private static bool ProcessStatement(Statement st, StatementBlock block)
{
if (st is AssignmentStatement)
{
var assign = (AssignmentStatement)st;
if (assign.Value is BinOpExpression)
{
var exp = (BinOpExpression)assign.Value;
if ((exp.Left is BinOpExpression || exp.Right is BinOpExpression) &&
exp.Left != assign.Target)
{
block.Statements.Add(new AssignmentStatement {
Target = assign.Target,
Value = exp.Left
});
block.Statements.Add(new AssignmentStatement {
Target = assign.Target,
Value = new BinOpExpression {
Left = assign.Target,
Operation = exp.Operation,
Right = exp.Right
}
});
return(true);
}
}
}
block.Statements.Add(st);
return(false);
}
/// <summary>
/// Processes the statement with specified cpu context. Returns false if we couldn't read due to invalid memory address.
/// </summary>
/// <param name="cpu">The cpu.</param>
/// <param name="parent">The parent.</param>
/// <param name="stack">The stack.</param>
/// <returns></returns>
internal override bool Process(CPURegisters cpu, StatementBlock parent, List <IntPtr> stack)
{
var ptr = this.Func(cpu);
stack.Add(ptr);
return(true);
}
public void ShouldParseDoEnd()
{
var expected =
new StatementBlock(new DoEndBlock(new StatementBlock(new EmptyStatement(), new EmptyStatement())));
var actual = SyntaxParser.Parse("do ; ; end");
Assert.AreEqual(expected,actual);
}
public static void Run(StatementBlock block)
{
foreach (var st in block.Statements)
{
ProcessStatement(st);
}
}
public static void Run(StatementBlock block, RandomGenerator random)
{
var context = new TransformContext
{
Statements = block.Statements.ToArray(),
Usages = block.Statements.ToDictionary(s => s, s => GetVariableUsage(s).ToArray()),
Definitions = block.Statements.ToDictionary(s => s, s => GetVariableDefinition(s).ToArray())
};
for (int i = 0; i < ITERATION; i++)
{
foreach (Statement st in context.Statements)
{
int index = block.Statements.IndexOf(st);
Variable[] vars = GetVariableUsage(st).Concat(GetVariableDefinition(st)).ToArray();
// Statement can move between defIndex & useIndex without side effects
int defIndex = SearchUpwardKill(context, st, block, index);
int useIndex = SearchDownwardKill(context, st, block, index);
// Move to a random spot in the interval
int newIndex = defIndex + random.NextInt32(1, useIndex - defIndex);
if (newIndex > index)
{
newIndex--;
}
block.Statements.RemoveAt(index);
block.Statements.Insert(newIndex, st);
}
}
}
public void ShouldParseDoEnd()
{
var expected =
new StatementBlock(new DoEndBlock(new StatementBlock(new EmptyStatement(), new EmptyStatement())));
var actual = SyntaxParser.Parse("do ; ; end");
Assert.AreEqual(expected, actual);
}
public void ShouldParseLocalAssignment()
{
var assignment = new Assignment(new[] {new Variable("a")},
new[] {new ConstantExpression(Constants.One)}, true);
var expected = new StatementBlock(assignment);
var actual = SyntaxParser.Parse("local a = 1");
Assert.AreEqual(expected, actual);
}
public void ShouldParseSimpleFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new[] {"arg1", "arg2"},
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func(arg1,arg2) ; end");
Assert.AreEqual(expected, actual);
}
private static void PostProcessStatements(StatementBlock block, RandomGenerator random)
{
MulToShiftTransform.Run(block);
NormalizeBinOpTransform.Run(block);
ExpansionTransform.Run(block);
ShuffleTransform.Run(block, random);
ConvertVariables.Run(block);
}
public void ShouldParseNoArgumentsFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new string[0],
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func() ; end");
Assert.AreEqual(expected, actual);
}
public void ExceptionFilterErrors(
string document,
StatementBlock expectedStatement,
RazorError[] expectedErrors)
{
// Act & Assert
ParseBlockTest(document, expectedStatement, expectedErrors);
}
public void ShouldParseSimpleAssignment()
{
var assignment = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, false);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("a = 1"));
}
public void ShouldParseMultipleAssignments()
{
var assignment = new Assignment(new[] { new Variable("a"), new Variable("b") },
new[] { new ConstantExpression(Constants.One) }, true);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("local a,b = 1"));
}
public static void ShouldParseMultilineStringAssignment()
{
var expected = new StatementBlock(new Assignment(new Variable("a"),
new StringConstantExpression("TEST1\nTEST2"), false));
var actual = SyntaxParser.Parse("a = [[TEST1\nTEST2]]");
Assert.AreEqual(expected, actual);
}
public void ShouldParseIfThen()
{
var conditionExpression = new BracketedExpression(new ConstantExpression(Constants.False));
var emptyStatementBlock = new StatementBlock(new EmptyStatement());
var ifStatement = new IfStatement(conditionExpression, emptyStatementBlock);
var expected = new StatementBlock(ifStatement);
Assert.AreEqual(expected, SyntaxParser.Parse("if (false) then ; end"));
}
public void ShouldParseSequentialAssignments()
{
var assignment1 = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, true);
var assignment2 = new Assignment(new[] { new Variable("b") },
new[] { new ConstantExpression(Constants.Two) }, true);
var expected = new StatementBlock(assignment1, assignment2);
Assert.AreEqual(expected, SyntaxParser.Parse("local a = 1\nlocal b = 2"));
}
public void ShouldParseIfThen()
{
var conditionExpression = new BracketedExpression(new ConstantExpression(Constants.False));
var emptyStatementBlock = new StatementBlock(new EmptyStatement());
var ifStatement = new IfStatement(conditionExpression, emptyStatementBlock);
var expected = new StatementBlock(ifStatement);
Assert.AreEqual(expected, SyntaxParser.Parse("if (false) then ; end"));
}
public void ShouldParseSimpleFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new[] { "arg1", "arg2" },
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func(arg1,arg2) ; end");
Assert.AreEqual(expected, actual);
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
#region IBindToMethod Members
void IBindToMethod.BindToMethod(MethodMember method)
{
if (m_HomeStatementBlock == null)
{
m_HomeStatementBlock = method.Body;
}
method.RegisterLocal(this, out m_LocalIndex);
}
// for each back edge whose source is in a while loop, add the appropriate initializer of source to the InitializerSet
private void AddLoopInitializers(StatementBlock block, ICollection <NodeIndex> usedNodes, Dictionary <NodeIndex, StatementBlock> blockOfNode, DependencyGraph2 g)
{
if (block is Loop loop)
{
AddLoopInitializers(loop.tail, usedNodes, blockOfNode, g);
AddLoopInitializers(loop.firstIterPostBlock, usedNodes, blockOfNode, g);
}
AddLoopInitializers(block.indices, usedNodes, blockOfNode, g);
}
public void ShouldParseNoArgumentsFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new string[0],
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func() ; end");
Assert.AreEqual(expected, actual);
}
public static void Run(StatementBlock block)
{
var mainBuff = new Variable("{BUFFER}");
for (int i = 0; i < block.Statements.Count; i++)
{
block.Statements[i] = ReplaceVar(block.Statements[i], mainBuff);
}
}
public static byte[] GenerateBlockOffest(StatementBlock block)
{
using (BinaryWriter binaryWriter = new BinaryWriter(new MemoryStream()))
{
GenerateBlockOffest(binaryWriter, block);
binaryWriter.Flush();
return(((MemoryStream)binaryWriter.BaseStream).ToArray());
}
}
public static void ShouldParseEmptyInitializer()
{
var expected = new StatementBlock(new Assignment(new Variable("a"),
new TableInitializerExpression(),
false));
var actual = SyntaxParser.Parse("a = {}");
Assert.AreEqual(expected, actual);
}
public void ShouldParseWhileBlock()
{
var expected =
new StatementBlock(new WhileStatement(new ConstantExpression(Constants.True),
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("while true do ; end");
Assert.AreEqual(expected, actual);
}
public void ShouldParseLocalAssignment()
{
var assignment = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, true);
var expected = new StatementBlock(assignment);
var actual = SyntaxParser.Parse("local a = 1");
Assert.AreEqual(expected, actual);
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
void IBindToMethod.ResetBinding()
{
if (m_HomeStatementBlock != null && m_HomeStatementBlock == m_HomeStatementBlock.OwnerMethod.Body)
{
m_HomeStatementBlock = null;
}
m_LocalBuilder = null;
m_LocalIndex = -1;
}
//-------------------------------------------------------------------------------------------------------------------------------------------------
public OperandCapture(IScopedOperand sourceOperand, StatementBlock sourceOperandHome, IAnonymousMethodOperand consumerMethod)
{
this.SourceOperand = sourceOperand;
this.SourceOperandHome = sourceOperandHome;
m_Consumers = new HashSet <IAnonymousMethodOperand>()
{
consumerMethod
};
}
public void AnalyzeDependencies(AdjacencyGraph <string, IEdge <string> > graph)
{
P.AnalyzeDependencies(graph, Info, RefId);
foreach (var x in LocalMap.Values)
{
var varRefId = $"{RefId}|{x.CppName}";
P.AnalyzeDependencies(graph, varRefId, x.DataType);
}
StatementBlock.AnalyzeDependencies(graph);
}
public static Expression Analyze(AnalysisContext analysisContext, StatementBlock statementBlock)
{
var analyzer = new StatementBlockAnalyzer(analysisContext);
statementBlock.Accept(analyzer);
if (analyzer._result == null)
throw new InternalException("A StatementBlockAnalyzer did not generate a _result");
return analyzer._result;
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
public virtual void VisitStatementBlock(StatementBlock statementBlock)
{
if (statementBlock != null)
{
foreach (var statement in statementBlock)
{
statement.AcceptVisitor(this);
}
}
}
public void ShouldParseSequentialAssignments()
{
var assignment1 = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, true);
var assignment2 = new Assignment(new[] { new Variable("b") },
new[] { new ConstantExpression(Constants.Two) }, true);
var expected = new StatementBlock(assignment1, assignment2);
Assert.AreEqual(expected, SyntaxParser.Parse("local a = 1\nlocal b = 2"));
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
internal MethodMember(ClassType ownerClass, MethodFactoryBase methodFactory, ClosureDefinition closure)
: base(ownerClass, methodFactory.MemberName)
{
m_MethodFactory = methodFactory;
m_Closure = closure;
m_Writers = new List <MethodWriterBase>();
m_Statements = new StatementBlock();
m_Locals = new List <ILocal>();
m_TransparentWriter = new TransparentMethodWriter(this);
}
public override void BeforeVisit(StatementBlock node)
{
base.BeforeVisit(node);
_analysisContext.ScopeStack.Push(node.SymbolTable);
}
public override void AfterVisit(StatementBlock node)
{
base.AfterVisit(node);
_result = Expression.Block(node.SymbolTable.GetParameterExpressions(), _statements.Where(e => e != null));
_analysisContext.ScopeStack.Pop();
}
public ForStatement(IEnumerable<LuaExpression> conditions, StatementBlock body)
{
Conditions = conditions;
Body = body;
}
private List<StatementBlock> GetStatementBlocksRecursive(StatementBlock block)
{
List<StatementBlock> blocks = new List<StatementBlock>();
foreach (Statement statement in block.statements) {
if (statement.IsTypeOf<NamespaceDef>()) {
NamespaceDef ns = (NamespaceDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(ns.statementBlock));
blocks.Add(ns.statementBlock);
}
if (statement.IsTypeOf<TypeDef>()) {
TypeDef t = (TypeDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(t.statementBlock));
blocks.Add(t.statementBlock);
}
if (statement.IsTypeOf<StatementBlock>()) {
StatementBlock sb = (StatementBlock)statement;
blocks.AddRange(GetStatementBlocksRecursive(sb));
blocks.Add(sb);
}
if (statement.IsTypeOf<MethodDef>()) {
MethodDef m = (MethodDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(m.statementBlock));
blocks.Add(m.statementBlock);
}
if (statement.IsTypeOf<PropertyDef>()) {
PropertyDef p = (PropertyDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(p.statementBlock));
blocks.Add(p.statementBlock);
}
if (statement.IsTypeOf<ForDef>()) {
ForDef f = (ForDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(f.statementBlock));
blocks.Add(f.statementBlock);
}
if (statement.IsTypeOf<ForEachDef>()) {
ForEachDef f = (ForEachDef)statement;
blocks.AddRange(GetStatementBlocksRecursive(f.statementBlock));
blocks.Add(f.statementBlock);
}
}
return blocks;
}
public override void Exit(StatementBlock node)
{
level--;
}
public virtual void Exit(StatementBlock node)
{
}
public void ExceptionFilterErrors(
string document,
StatementBlock expectedStatement,
RazorError[] expectedErrors)
{
// Act & Assert
ParseBlockTest(document, expectedStatement, expectedErrors);
}
public override void BeforeVisit(StatementBlock node)
{
base.BeforeVisit(node);
_result = StatementBlockAnalyzer.Analyze(_analysisContext, node);
base.PushMode(VisitorMode.VisitNodeOnly);
}
public override void AfterVisit(StatementBlock node)
{
base.AfterVisit(node);
base.PopMode();
}
public void ParseBlock_WithUnexpectedTransitionsInAttributeValue_Throws()
{
// Arrange
var expected = new StatementBlock(
Factory.MetaCode("{").Accepts(AcceptedCharacters.None),
new MarkupBlock(
new MarkupTagBlock(
Factory.Markup("<span"),
new MarkupBlock(
new AttributeBlockChunkGenerator("foo", new LocationTagged<string>(" foo='", 6, 0, 6), new LocationTagged<string>("'", 15, 0, 15)),
Factory.Markup(" foo='").With(SpanChunkGenerator.Null),
new MarkupBlock(
new DynamicAttributeBlockChunkGenerator(new LocationTagged<string>(string.Empty, 12, 0, 12), 12, 0, 12),
new ExpressionBlock(
Factory.CodeTransition(),
Factory.EmptyCSharp().AsImplicitExpression(CSharpCodeParser.DefaultKeywords).Accepts(AcceptedCharacters.NonWhiteSpace))),
new MarkupBlock(
new DynamicAttributeBlockChunkGenerator(new LocationTagged<string>(" ", 13, 0, 13), 13, 0, 13),
Factory.Markup(" ").With(SpanChunkGenerator.Null),
new ExpressionBlock(
Factory.CodeTransition(),
Factory.EmptyCSharp().AsImplicitExpression(CSharpCodeParser.DefaultKeywords).Accepts(AcceptedCharacters.NonWhiteSpace))),
Factory.Markup("'").With(SpanChunkGenerator.Null)),
Factory.Markup(" />").Accepts(AcceptedCharacters.None))),
Factory.EmptyCSharp().AsStatement(),
Factory.MetaCode("}").Accepts(AcceptedCharacters.None));
var expectedErrors = new RazorError[]
{
new RazorError(
@"A space or line break was encountered after the ""@"" character. Only valid identifiers, keywords, comments, ""("" and ""{"" are valid at the start of a code block and they must occur immediately following ""@"" with no space in between.",
new SourceLocation(13, 0, 13),
length: 1),
new RazorError(
@"""' />}"" is not valid at the start of a code block. Only identifiers, keywords, comments, ""("" and ""{"" are valid.",
new SourceLocation(15, 0, 15),
length: 5),
};
// Act & Assert
ParseBlockTest("{<span foo='@ @' />}", expected, expectedErrors);
}
public virtual bool Enter(StatementBlock node)
{
return true;
}
public void ExceptionFilters(string document, StatementBlock expectedStatement)
{
// Act & Assert
ParseBlockTest(document, expectedStatement);
}
public override bool Enter(StatementBlock node)
{
Print("StatementBlock");
level++;
return true;
}
public virtual object Walk(StatementBlock node)
{
if (Enter(node))
{
foreach (var statement in node.Statements)
{
statement.Accept(this);
}
}
Exit(node);
return null;
}
public void ParseBlock_WithDoubleTransition_EndOfFile_Throws()
{
// Arrange
var expected = new StatementBlock(
Factory.MetaCode("{").Accepts(AcceptedCharacters.None),
new MarkupBlock(
new MarkupTagBlock(
Factory.Markup("<span"),
new MarkupBlock(
new AttributeBlockChunkGenerator("foo", new LocationTagged<string>(" foo='", 6, 0, 6), new LocationTagged<string>(string.Empty, 14, 0, 14)),
Factory.Markup(" foo='").With(SpanChunkGenerator.Null),
new MarkupBlock(
Factory.Markup("@").With(new LiteralAttributeChunkGenerator(new LocationTagged<string>(string.Empty, 12, 0, 12), new LocationTagged<string>("@", 12, 0, 12))).Accepts(AcceptedCharacters.None),
Factory.Markup("@").With(SpanChunkGenerator.Null).Accepts(AcceptedCharacters.None)))),
Factory.EmptyHtml()));
var expectedErrors = new RazorError[]
{
new RazorError(
@"End of file or an unexpected character was reached before the ""span"" tag could be parsed. Elements inside markup blocks must be complete. They must either be self-closing (""<br />"") or have matching end tags (""<p>Hello</p>""). If you intended to display a ""<"" character, use the ""<"" HTML entity.",
new SourceLocation(2, 0, 2),
length: 4),
new RazorError(
@"The code block is missing a closing ""}"" character. Make sure you have a matching ""}"" character for all the ""{"" characters within this block, and that none of the ""}"" characters are being interpreted as markup.",
SourceLocation.Zero,
length: 1),
};
// Act & Assert
ParseBlockTest("{<span foo='@@", expected, expectedErrors);
}
public override object Walk(StatementBlock node)
{
object result = null;
Action action = delegate
{
foreach (var statement in node.Statements)
{
result = statement.Accept(this);
}
};
Context.OpenScopeFor(action);
return result;
}
public StatementBlock ParseStatementBlock()
{
var block = new StatementBlock { Token = Next() };
Match(TokenType.LeftBrace);
while (Next().IsNot(TokenType.RightBrace))
block.Statements.Add(ParseStatement());
Match(TokenType.RightBrace);
return block;
}
public void ShouldParseWhileBlock()
{
var expected = new StatementBlock(new WhileStatement(new ConstantExpression(Constants.True),new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("while true do ; end");
Assert.AreEqual(expected,actual);
}
