private CompiledMethod ParseAndCompileSingleMethod(string source)
{
var sourceBytes = Encoding.UTF8.GetBytes(source);
var diagnostics = new BenchmarkDiagnosticsSink();
// Parse the source
const string sourceFilename = "test.cle";
var syntaxTree = SyntaxParser.Parse(sourceBytes.AsMemory(), sourceFilename, diagnostics);
if (syntaxTree is null || syntaxTree.Functions.Count != 1)
{
throw new InvalidOperationException("Expected a single method");
}
// Compile the declaration
var declarationProvider = new NullDeclarationProvider();
var declaration = MethodDeclarationCompiler.Compile(syntaxTree.Functions[0],
syntaxTree.Namespace, sourceFilename,
0, declarationProvider, diagnostics);
// Compile the method body
var result = new MethodCompiler(declarationProvider, diagnostics)
.CompileBody(syntaxTree.Functions[0], declaration !, syntaxTree.Namespace, sourceFilename);
if (diagnostics.DiagnosticCount > 0)
{
throw new InvalidOperationException("Expected no diagnostics");
}
return(result !);
}
public void Parse(string identifier)
{
this.classificationSpans.Clear();
var syntaxParser = new SyntaxParser(this);
syntaxParser.Parse(identifier);
}
public void MultiLineComments()
{
string sText =
@"/*Комментарий create
create
-- function (*/x/**/function/*
xxx";
List <string> lines = this.GetLines(sText);
SyntaxParser oParser = new SyntaxParser(new MsSqlSyntaxSettings(), 4);
List <Token> tokens = oParser.Parse(lines);
List <Token> expectedTokens = new List <Token>(new Token[]
{
new Token("/*Комментарий create ", new TextPoint(0, 0, 0), new TextPoint(0, 21, 21), new TextStyle("MultiLine Comment", Color.Black)),
new Token("create", new TextPoint(1, 0, 0), new TextPoint(1, 6, 6), new TextStyle("MultiLine Comment", Color.Black)),
new Token("-- function (*/", new TextPoint(2, 0, 0), new TextPoint(2, 15, 15), new TextStyle("MultiLine Comment", Color.Black)),
new Token("x", new TextPoint(2, 15, 15), new TextPoint(2, 16, 16), new TextStyle("Normal", Color.Black)),
new Token("/**/", new TextPoint(2, 16, 16), new TextPoint(2, 20, 20), new TextStyle("MultiLine Comment", Color.Black)),
new Token("function", new TextPoint(2, 20, 20), new TextPoint(2, 28, 28), new TextStyle("Keywords1", Color.Black)),
new Token("/*", new TextPoint(2, 28, 28), new TextPoint(2, 30, 30), new TextStyle("MultiLine Comment", Color.Black)),
new Token(" xxx", new TextPoint(3, 0, 0), new TextPoint(3, 4, 4), new TextStyle("MultiLine Comment", Color.Black))
});
for (int i = 0; i < tokens.Count; i++)
{
Token oToken1 = tokens[i];
Token oToken2 = expectedTokens[i];
Assert.AreEqual(oToken1.Text, oToken2.Text);
Assert.AreEqual(oToken1.Style.Name, oToken2.Style.Name);
Assert.AreEqual(oToken1.Start.ToString(), oToken2.Start.ToString());
Assert.AreEqual(oToken1.End.ToString(), oToken2.End.ToString());
}
}
void Test(string source, string expectedResult)
{
try
{
Tokenizer lexer = new Tokenizer(new StringReader(source), "TokenizerTest");
SyntaxParser parser = new SyntaxParser();
System.Text.StringBuilder sb = new System.Text.StringBuilder();
bool addSpace = false;
do
{
SObject result = SyntaxParser.Parse(lexer);
if (result == null)
{
break;
}
if (addSpace)
{
sb.Append(" ");
}
else
{
addSpace = true;
}
sb.Append(Inspector.InspectObject(result));
} while (lexer.LastToken != null);
string sresult = sb.ToString();
Assert.AreEqual(expectedResult, sresult);
}
catch (BaseSchemeError ex)
{
Debug.LogError(string.Format("Source: {0}\n{1}\n{2}", source, ex.Message, ex.StackTrace));
}
}
public ParseResult <IN, OUT> ParseWithContext(IList <Token <IN> > tokens, object parsingContext = null, string startingNonTerminal = null)
{
var result = new ParseResult <IN, OUT>();
var cleaner = new SyntaxTreeCleaner <IN>();
var syntaxResult = SyntaxParser.Parse(tokens, startingNonTerminal);
syntaxResult = cleaner.CleanSyntaxTree(syntaxResult);
if (!syntaxResult.IsError && syntaxResult.Root != null)
{
var r = Visitor.VisitSyntaxTree(syntaxResult.Root, parsingContext);
result.Result = r;
result.SyntaxTree = syntaxResult.Root;
result.IsError = false;
}
else
{
result.Errors = new List <ParseError>();
var unexpectedTokens = syntaxResult.Errors.Cast <UnexpectedTokenSyntaxError <IN> >().ToList();
var byEnding = unexpectedTokens.GroupBy(x => x.UnexpectedToken.Position).OrderBy(x => x.Key);
var errors = new List <ParseError>();
foreach (var expecting in byEnding)
{
var expectingTokens = expecting.SelectMany(x => x.ExpectedTokens).Distinct();
var expected = new UnexpectedTokenSyntaxError <IN>(expecting.First().UnexpectedToken, expectingTokens.ToArray());
errors.Add(expected);
}
result.Errors.AddRange(errors);
result.IsError = true;
}
return(result);
}
/// <summary>
/// Parses the given source text and returns the syntax tree if successful.
/// </summary>
protected SourceFileSyntax?ParseSource(string source, out TestingDiagnosticSink diagnostics)
{
var sourceBytes = Encoding.UTF8.GetBytes(source);
diagnostics = new TestingDiagnosticSink();
return(SyntaxParser.Parse(sourceBytes.AsMemory(), "test.cle", diagnostics));
}
public void ShouldParseDoEnd()
{
var expected =
new StatementBlock(new DoEndBlock(new StatementBlock(new EmptyStatement(), new EmptyStatement())));
var actual = SyntaxParser.Parse("do ; ; end");
Assert.AreEqual(expected, actual);
}
public static void ShouldParseMultilineStringAssignment()
{
var expected = new StatementBlock(new Assignment(new Variable("a"),
new StringConstantExpression("TEST1\nTEST2"), false));
var actual = SyntaxParser.Parse("a = [[TEST1\nTEST2]]");
Assert.AreEqual(expected, actual);
}
public void ShouldParseSimpleAssignment()
{
var assignment = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, false);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("a = 1"));
}
public void ShouldParseMultipleAssignments()
{
var assignment = new Assignment(new[] { new Variable("a"), new Variable("b") },
new[] { new ConstantExpression(Constants.One) }, true);
var expected = new StatementBlock(assignment);
Assert.AreEqual(expected, SyntaxParser.Parse("local a,b = 1"));
}
public void Parse_Basic()
{
var parser = new SyntaxParser();
var result = parser.Parse(new Source(@"{
var test=System.String.test;
var test2=System.String.test();
}", Method.EntryInfo));
}
public int ParseSingleFileWithoutErrors()
{
var diagnostics = new BenchmarkDiagnosticsSink();
SyntaxParser.Parse(_source, "filename.cle", diagnostics);
return(diagnostics.DiagnosticCount);
}
public void ShouldParseLocalAssignment()
{
var assignment = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, true);
var expected = new StatementBlock(assignment);
var actual = SyntaxParser.Parse("local a = 1");
Assert.AreEqual(expected, actual);
}
public void ShouldParseIfThen()
{
var conditionExpression = new BracketedExpression(new ConstantExpression(Constants.False));
var emptyStatementBlock = new StatementBlock(new EmptyStatement());
var ifStatement = new IfStatement(conditionExpression, emptyStatementBlock);
var expected = new StatementBlock(ifStatement);
Assert.AreEqual(expected, SyntaxParser.Parse("if (false) then ; end"));
}
private void ParseAllText()
{
Text = Text.Replace("\t", Tab);
if (SyntaxParser != null)
{
SyntaxParser.Parse(Text, 0, Text.Length);
}
InvalidateVisual();
}
public static void ShouldParseEmptyInitializer()
{
var expected = new StatementBlock(new Assignment(new Variable("a"),
new TableInitializerExpression(),
false));
var actual = SyntaxParser.Parse("a = {}");
Assert.AreEqual(expected, actual);
}
public void ShouldParseSimpleFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new[] { "arg1", "arg2" },
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func(arg1,arg2) ; end");
Assert.AreEqual(expected, actual);
}
public void ShouldParseNoArgumentsFunctionDeclaration()
{
var expected = new StatementBlock(new FunctionDeclarationStatement("func", new string[0],
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("function func() ; end");
Assert.AreEqual(expected, actual);
}
public void ShouldParseWhileBlock()
{
var expected =
new StatementBlock(new WhileStatement(new ConstantExpression(Constants.True),
new StatementBlock(new EmptyStatement())));
var actual = SyntaxParser.Parse("while true do ; end");
Assert.AreEqual(expected, actual);
}
public void ShouldNotFailOnNoCode()
{
var parser = new SyntaxParser();
var result = parser.Parse(string.Empty);
result.TypeDeclarations.Any().ShouldEqual(false);
result.MethodPrototypes.Any().ShouldEqual(false);
result.MethodExpressions.Any().ShouldEqual(false);
result.Evaluations.ShouldEqual(string.Empty);
}
public void ShouldParseSequentialAssignments()
{
var assignment1 = new Assignment(new[] { new Variable("a") },
new[] { new ConstantExpression(Constants.One) }, true);
var assignment2 = new Assignment(new[] { new Variable("b") },
new[] { new ConstantExpression(Constants.Two) }, true);
var expected = new StatementBlock(assignment1, assignment2);
Assert.AreEqual(expected, SyntaxParser.Parse("local a = 1\nlocal b = 2"));
}
protected override void OnTextChanged(TextChangedEventArgs e)
{
base.OnTextChanged(e);
Text = Text.Replace("\t", Tab);
if (SyntaxParser != null)
{
SyntaxParser.Parse(Text, 0, Text.Length);
}
InvalidateVisual();
}
public void ShouldExtractEvaluation()
{
const string Code = "void Foo() {} int x; void Bar() {}";
var parser = new SyntaxParser();
var result = parser.Parse(Code);
result.Evaluations.ShouldContain("int x;");
result.Evaluations.ShouldNotContain("Foo");
result.Evaluations.ShouldNotContain("Bar");
}
public static EntryNode Transform(this IList <SyntaxToken> tokens)
{
var transformer = new SyntaxParser(tokens);
var result = transformer.Parse() as EntryNode;
if (transformer.Errors.Count > 0)
{
throw new Exception(string.Join(Environment.NewLine, transformer.Errors.ToArray()));
}
return(result);
}
public static EntryNode Parse(this IList <SyntaxToken> tokens, out IList <string> errors)
{
var parser = new SyntaxParser(tokens);
var result = parser.Parse();
errors = parser.Errors;
if (errors.Count > 0)
{
throw new Exception(string.Join(Environment.NewLine, errors.ToArray()));
}
return(result as EntryNode);
}
public void WhenStringStartsWithGerkinSyntaxThenCallsObserverWithOneGivenAndMultiplePascalSpans()
{
var mockGherkinSyntaxParserObserver = Substitute.For<ISyntaxParserObserver>();
var gherkinSyntaxParser = new SyntaxParser(mockGherkinSyntaxParserObserver);
gherkinSyntaxParser.Parse("GivenTheAccountIsInCredit");
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(0, 5); // Given
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(5, 3); // The
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(8, 7); // Account
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(15, 2); // Is
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(17, 2); // In
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(19, 6); // Credit
}
public void ShouldParseSimpleStatement()
{
const string Code = "var x = 42;";
var parser = new SyntaxParser();
var result = parser.Parse(Code);
result.TypeDeclarations.Any().ShouldEqual(false);
result.MethodPrototypes.Any().ShouldEqual(false);
result.MethodExpressions.Any().ShouldEqual(false);
result.Evaluations.ShouldEqual(Code);
}
/// <summary>
/// Creates a confrolflow graph from script with given file name.
/// </summary>
/// <param name="file">File</param>
/// <returns>new instace of ControlFlowGraph</returns>
public static ControlFlowGraph FromFile(FileInfo file)
{
var fileName = file.FullName;
SyntaxParser parser = GenerateParser(fileName);
parser.Parse();
if (parser.Ast == null)
{
throw new ArgumentException("The specified file cannot be parsed.");
}
return(new ControlFlowGraph(parser.Ast, file));
}
/// <summary>
/// Creates a confrolflow graph from fileName in parameter.
/// </summary>
/// <param name="phpCode">source code in string</param>
/// <param name="fileName">name of the file</param>
/// <returns>new instace of ControlFlowGraph</returns>
public static ControlFlowGraph FromSource(string phpCode, string fileName)
{
PhpSourceFile source_file = new PhpSourceFile(new FullPath(Path.GetDirectoryName(fileName)), new FullPath(fileName));
SyntaxParser parser = new SyntaxParser(source_file, phpCode);
parser.Parse();
if (parser.Ast == null)
{
throw new ArgumentException("The specified input cannot be parsed.");
}
return(new ControlFlowGraph(parser.Ast, new FileInfo(fileName)));
}
public static void ShouldParseMultiValueInitializer()
{
var expected = new StatementBlock(new Assignment(new Variable("a"),
new TableInitializerExpression(
new VariableExpression(new Variable("a")),
new StringConstantExpression("b"),
new ConstantExpression(Constants.Two)
),
false));
var actual = SyntaxParser.Parse("a = {a,\"b\",2}");
Assert.AreEqual(expected, actual);
}
static SyntaxParser InitializeParser(string phpCode, string fileName)
{
PhpSourceFile source_file = new PhpSourceFile(new FullPath(Path.GetDirectoryName(fileName)), new FullPath(fileName));
SyntaxParser parser = new SyntaxParser(source_file, phpCode);
parser.Parse();
if (parser.Ast == null)
{
throw new ArgumentException("The specified input cannot be parsed.");
}
return(parser);
}
public void WhenStringStartsWithAndContainsGherkinSntaxThenCallsObserverWithOneGivenAndOneAndSpan()
{
var mockGherkinSyntaxParserObserver = Substitute.For<ISyntaxParserObserver>();
var gherkinSyntaxParser = new SyntaxParser(mockGherkinSyntaxParserObserver);
gherkinSyntaxParser.Parse("GivenTheAccountIsInCreditAndTheCardIsValid");
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(0, 5); // Given
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(5, 3); // The
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(8, 7); // Account
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(15, 2); // Is
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(17, 2); // In
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(19, 6); // Credit
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(25, 3); // And
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(28, 3); // The
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(31, 4); // Card
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(35, 2); // Is
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(37, 5); // Valid
}
public void WhenStringStartWithAndContainsGherkinSntaxThenCallsObserverWithOneWhenAndOneThenSpan()
{
var mockGherkinSyntaxParserObserver = Substitute.For<ISyntaxParserObserver>();
var gherkinSyntaxParser = new SyntaxParser(mockGherkinSyntaxParserObserver);
gherkinSyntaxParser.Parse("WhenCustomerRequestsCashThenEnsureAccountIsDebitedAndEnsureCashDispensed");
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(0, 4); // When
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(4, 8); // Customer
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(12, 8); // Requests
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(20, 4); // Cash
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(24, 4); // Then
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(28, 6); // Ensure
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(34, 7); // Account
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(41, 2); // Is
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(43, 7); // Debited
mockGherkinSyntaxParserObserver.Received(1).AddGherkinSyntaxSpanAt(50, 3); // And
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(53, 6); // Ensure
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(59, 4); // Cash
mockGherkinSyntaxParserObserver.Received(1).AddPascalCaseSpanAt(63, 9); // Dispensed
}
public void Parse(string identifier)
{
this.classificationSpans.Clear();
var syntaxParser = new SyntaxParser(this);
syntaxParser.Parse(identifier);
}