public void Simplify_PhpCodeWithConstants_ConstantsFolded()
{
var sourceCodeRep = new MemoryCodeRepository(
"<?php\r\n" +
"echo 'Hello ' . 'World' . '!';\r\n" +
"echo 60 * 60 * 24;\r\n" +
"echo 6 + 6 * 6;\r\n" +
"$a = -3;\r\n" +
"$b = -3.1;",
"constants.php"
);
var logger = new LoggerMessageCounter();
var workflow = new Workflow(sourceCodeRep, stage: Stage.Ust)
{
IsSimplifyUst = true,
Logger = logger
};
workflow.Process();
Assert.IsTrue(logger.ContainsDebugMessagePart("Hello World!"));
Assert.IsTrue(logger.ContainsDebugMessagePart("86400"));
Assert.IsTrue(logger.ContainsDebugMessagePart("42"));
Assert.IsTrue(logger.ContainsDebugMessagePart("-3"));
Assert.IsTrue(logger.ContainsDebugMessagePart("-3.1"));
}
public void Parse_SyntaxErrorFileCSharp_CatchErrors()
{
var logger = new LoggerMessageCounter();
TestHelper.CheckFile("ParseError.cs", Language.CSharp, Stage.Parse, logger, true);
Assert.AreEqual(7, logger.ErrorCount);
}
public void Preprocess_MultiMultiPattern_RemovedDuplicates()
{
UstNode patternWithDuplicateMultiStatementsExpressions = new PatternStatements
{
Statements = new List <Statement>()
{
new ExpressionStatement(
new InvocationExpression
{
Target = new IdToken("test_call"),
Arguments = new PatternExpressions
{
Collection = new List <Expression>
{
new IdToken("a"),
new IdToken("b"),
new PatternMultipleExpressions(),
new PatternMultipleExpressions(),
new IdToken("z")
}
}
}),
new PatternMultipleStatements(),
new PatternMultipleStatements(),
new ExpressionStatement(
new VariableDeclarationExpression
{
Type = new TypeToken("int"),
Variables = new List <AssignmentExpression>()
{
new AssignmentExpression
{
Left = new IdToken("a"),
Right = new IntLiteral {
Value = 42
}
}
}
}
)
}
};
var logger = new LoggerMessageCounter();
var processor = new DslProcessor();
UstPreprocessor preprocessor = new UstPreprocessor()
{
Logger = logger
};
UstNode result = preprocessor.Preprocess(patternWithDuplicateMultiStatementsExpressions);
Assert.AreEqual(1, result.GetAllDescendants().Count(child => child.NodeType == NodeType.PatternMultipleStatements));
Assert.AreEqual(1, result.GetAllDescendants().Count(child => child.NodeType == NodeType.PatternMultipleExpressions));
}
public void Convert_JavaPatternsWithErrors_MatchedResultsEqual()
{
var patternsLogger = new LoggerMessageCounter();
TestUtility.CheckFile("Patterns.java", Stage.Match, patternsLogger);
var patternWithErrorsLogger = new LoggerMessageCounter();
TestUtility.CheckFile("PatternsWithParseErrors.java", Stage.Match, patternWithErrorsLogger, true);
Assert.AreEqual(-1, patternWithErrorsLogger.InfoMessageCount - patternsLogger.InfoMessageCount);
}
public void Parse_Dsl_WithoutErrors(string fileName)
{
var data = File.ReadAllText(Path.Combine(TestHelper.TestsDataPath, fileName));
var logger = new LoggerMessageCounter();
var processor = new DslProcessor()
{
Logger = logger
};
UstNode result = processor.Deserialize(data, LanguageExt.AllPatternLanguages);
Assert.AreEqual(0, logger.ErrorCount);
}
public void Convert_PhpPatternsWithErrors_MatchedResultsEqual()
{
var patternsLogger = new LoggerMessageCounter();
TestHelper.CheckFile("Patterns.php", Language.Php, Stage.Match, patternsLogger);
var patternWithErrorsLogger = new LoggerMessageCounter();
TestHelper.CheckFile("PatternsWithParseErrors.php", Language.Php, Stage.Match, patternWithErrorsLogger, true);
Assert.AreEqual(patternsLogger.InfoMessageCount, patternWithErrorsLogger.InfoMessageCount);
}
public void ProcessDsl_SampleWithSyntaxError_HandleErrors()
{
var logger = new LoggerMessageCounter();
var data = "(?i)password(?-i)]> = <[\"\\w*\" || null]>";
var processor = new DslProcessor()
{
Logger = logger
};
UstNode result = processor.Deserialize(data, LanguageExt.AllPatternLanguages);
Assert.AreEqual(5, logger.ErrorCount);
}
private void Check(string patternsString)
{
var logger = new LoggerMessageCounter();
var patternsRepository = RepositoryFactory.CreatePatternsRepository(patternsString, null, logger);
var workflow = new Workflow(codeRepository)
{
PatternsRepository = patternsRepository
};
var result = workflow.Process();
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
Assert.AreEqual(1, result.MatchResults.Count);
}
public void Parse_Dsl_WithoutErrors(string fileName)
{
string data = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, fileName));
var logger = new LoggerMessageCounter();
var processor = new DslProcessor()
{
Logger = logger
};
PatternRoot result = processor.Deserialize(new CodeFile(data)
{
IsPattern = true
});
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
}
public void ProcessDsl_SampleWithSyntaxError_HandleErrors()
{
var logger = new LoggerMessageCounter();
var data = "(?i)password(?-i)]> = <[\"\\w*\" || null]>";
var processor = new DslProcessor()
{
Logger = logger
};
PatternRoot result = processor.Deserialize(new CodeFile(data)
{
IsPattern = true
});
Assert.AreEqual(5, logger.ErrorCount);
}
public void Sort_PatternVars_CorrectOrder()
{
var unsorted = new PatternOr
(
new PatternStringLiteral("42"),
new PatternIntLiteral(100),
new PatternIntLiteral(42),
new PatternIntLiteral(0),
new PatternStringLiteral("42"),
new PatternNot(new PatternStringLiteral("42")),
new PatternStringLiteral("Hello World!"),
new PatternIdToken("testId"),
new PatternIdToken("42"),
new PatternNot(new PatternStringLiteral("42"))
);
var expectedSorted = new PatternOr
(
new PatternIdToken("42"),
new PatternIdToken("testId"),
new PatternIntLiteral(0),
new PatternIntLiteral(42),
new PatternIntLiteral(100),
new PatternNot(new PatternStringLiteral("42")),
new PatternNot(new PatternStringLiteral("42")),
new PatternStringLiteral("42"),
new PatternStringLiteral("42"),
new PatternStringLiteral("Hello World!")
);
var logger = new LoggerMessageCounter();
var processor = new DslProcessor();
var normalizer = new PatternNormalizer()
{
Logger = logger
};
var actualPattern = (PatternOr)normalizer.Visit(unsorted);
List <PatternUst> actualAlternatives = actualPattern.Patterns;
List <PatternUst> expectedAlternatives = expectedSorted.Patterns;
Assert.AreEqual(expectedAlternatives.Count, actualAlternatives.Count);
for (int i = 0; i < expectedAlternatives.Count; i++)
{
Assert.IsTrue(expectedAlternatives[i].Equals(actualAlternatives[i]),
$"Not equal at {i} index: expected {expectedAlternatives[i]} not equals to {actualAlternatives[i]}");
}
}
public void Simplify_MultiMultiPattern_RemovedDuplicates()
{
var input = new PatternStatements
{
Statements = new List <PatternUst>()
{
new PatternInvocationExpression
{
Target = new PatternIdToken("test_call"),
Arguments = new PatternArgs
(
new PatternIdToken("a"),
new PatternIdToken("b"),
new PatternMultipleExpressions(),
new PatternMultipleExpressions(),
new PatternIdToken("z")
)
},
new PatternVarOrFieldDeclaration
{
Type = new PatternIdToken("int"),
Assignment = new PatternAssignmentExpression
{
Left = new PatternIdToken("a"),
Right = new PatternIntLiteral(42)
}
}
}
};
var logger = new LoggerMessageCounter();
var processor = new DslProcessor();
var normalizer = new PatternNormalizer()
{
Logger = logger
};
PatternUst result = normalizer.Visit(input);
var statements = ((PatternStatements)result).Statements;
var invocation = (PatternInvocationExpression)statements.ElementAt(0);
Assert.AreEqual(1, ((PatternArgs)invocation.Arguments).Args.Count(child => child is PatternMultipleExpressions));
}
public void Preprocess_CodeWithConstants_ConstantsFolded()
{
var sourceCodeRep = new MemoryCodeRepository(
"<?php\r\n" +
"echo 'Hello ' . 'World' . '!';\r\n" +
"echo 60 * 60 * 24;\r\n" +
"echo 6 + 6 * 6;"
);
var logger = new LoggerMessageCounter();
var workflow = new Workflow(sourceCodeRep, Language.Php, stage: Stage.Preprocess);
workflow.IsIncludePreprocessing = true;
workflow.Logger = logger;
workflow.Process();
Assert.IsTrue(logger.ContainsDebugMessagePart("Hello World!"));
Assert.IsTrue(logger.ContainsDebugMessagePart("86400"));
Assert.IsTrue(logger.ContainsDebugMessagePart("42"));
}
public void Process_Dsl_EqualsToHardcoded(string fileName)
{
var data = File.ReadAllText(Path.Combine(TestHelper.TestsDataPath, fileName));
var logger = new LoggerMessageCounter();
var processor = new DslProcessor()
{
Logger = logger, PatternExpressionInsideStatement = false
};
UstNode result = processor.Deserialize(data, fileName == @"DebugInfo.aipm" ? LanguageFlags.Php : LanguageExt.AllPatternLanguages);
Assert.AreEqual(0, logger.ErrorCount);
string patternName = Path.GetFileNameWithoutExtension(fileName);
Pattern defaultPattern = patterns.FirstOrDefault(p => p.DebugInfo.StartsWith(patternName));
if (defaultPattern == null)
{
Assert.Inconclusive($"Pattern {patternName} does not exists in DefaultPatternRepository");
}
Assert.IsTrue(result.Equals(defaultPattern.Data));
}
public void Reduct_BinaryAssignmentOperations_Reducted()
{
var sourceCodeRepositories = new Dictionary <Language, string>()
{
[Language.CSharp] =
@"class Test
{
void Main()
{
int a = 40;
a += 2;
}
}",
[Language.Java] =
@"class Test
{
void main()
{
int a = 40;
a += 2;
}
}",
[Language.Php] =
@"<?php
$a = 40;
$a += 2;",
};
foreach (var repository in sourceCodeRepositories)
{
var logger = new LoggerMessageCounter();
var workflow = new Workflow(new MemoryCodeRepository(repository.Value), repository.Key, stage: Stage.Preprocess);
workflow.Logger = logger;
workflow.Stage = Stage.Convert;
WorkflowResult workflowResult = workflow.Process();
Assert.IsTrue(workflowResult.Usts.First().Root.ToString().Contains("a = a + 2"));
}
}
public void Process_Dsl_EqualsToHardcoded(string fileName)
{
string data = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, fileName));
var logger = new LoggerMessageCounter();
var processor = new DslProcessor()
{
Logger = logger, PatternExpressionInsideStatement = false
};
PatternRoot result = processor.Deserialize(new CodeFile(data)
{
IsPattern = true
});
if (fileName == "DebugInfo.pattern")
{
new HashSet <Language>()
{
Php.Language
};
}
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
string patternName = Path.GetFileNameWithoutExtension(fileName);
PatternRoot defaultPattern = patterns.FirstOrDefault(p => p.DebugInfo.StartsWith(patternName));
if (defaultPattern == null)
{
Assert.Inconclusive($"Pattern {patternName} does not exists in DefaultPatternRepository");
}
var patternNormalizer = new PatternNormalizer();
defaultPattern = patternNormalizer.Normalize(defaultPattern);
Assert.AreEqual(defaultPattern.Node, result.Node);
}
public ParserUnit(ILanguageParser parser, LoggerMessageCounter logger, Task task)
{
Parser = parser;
Logger = logger;
Task = task;
}
public TestsCliProcessor()
{
Logger = new LoggerMessageCounter();
}
public override Language?Detect(string sourceCode, Language[] languages = null)
{
List <Language> langs = (languages ?? LanguageExt.Languages).ToList();
// Any PHP file contains start tag.
if (!sourceCode.Contains("<?"))
{
langs.Remove(Language.Php);
}
// Aspx and html code contains at least one tag.
if (!openTagRegex.IsMatch(sourceCode) || !closeTagRegex.IsMatch(sourceCode))
{
langs.Remove(Language.Aspx);
langs.Remove(Language.Html);
}
var sourceCodeFile = new SourceCodeFile("Temp Source Code")
{
Code = sourceCode
};
var parseUnits = new Queue <Tuple <Language, ParserUnit> >(langs.Count);
langs = langs
.GroupBy(l => ParserConverterBuilder.GetParserConverterSet(l).Parser.Language)
.Select(l => l.First())
.ToList();
if (langs.Count == 1)
{
return(langs[0]);
}
foreach (var language in langs)
{
var logger = new LoggerMessageCounter();
ILanguageParser languageParser = ParserConverterBuilder.GetParserConverterSet(language).Parser;
var task = Task.Factory.StartNew(() =>
{
languageParser.Logger = logger;
languageParser.Parse(sourceCodeFile);
});
parseUnits.Enqueue(Tuple.Create(language, new ParserUnit(languageParser, logger, task)));
}
int minErrorCount = int.MaxValue;
Language?resultWithMinErrors = null;
// Check every parseUnit completion every 50 ms.
while (parseUnits.Count > 0)
{
var pair = parseUnits.Dequeue();
if (!pair.Item2.Task.IsCompleted)
{
parseUnits.Enqueue(pair);
Thread.Sleep(50);
continue;
}
if (pair.Item2.Logger.ErrorCount == 0 && pair.Item1 != Language.Aspx)
{
return(pair.Item1);
}
var errorCount = pair.Item2.ParseErrorCount;
if (errorCount < minErrorCount)
{
minErrorCount = errorCount;
resultWithMinErrors = pair.Item1;
}
}
return(resultWithMinErrors);
}
public void Sort_PatternVars()
{
var unsortedExpressions = new List <Expression>()
{
new IntLiteral {
Value = 100
},
new IntLiteral {
Value = 42
},
new IntLiteral {
Value = 0
},
new StringLiteral {
Text = "42"
},
new StringLiteral {
Text = "Hello World!"
},
new IdToken {
Id = "testId"
},
new IdToken {
Id = "42"
},
new PatternExpression(new StringLiteral {
Text = "42"
}, true),
};
var expectedSortedExpressions = new List <Expression>
{
new StringLiteral {
Text = "42"
},
new PatternExpression(new StringLiteral {
Text = "42"
}, true),
new StringLiteral {
Text = "Hello World!"
},
new IdToken {
Id = "42"
},
new IdToken {
Id = "testId"
},
new IntLiteral {
Value = 0
},
new IntLiteral {
Value = 42
},
new IntLiteral {
Value = 100
},
};
var patternVarDef = new PatternVarDef
{
Id = "testVarDef",
Values = unsortedExpressions
};
var patternVars = new PatternNode
{
Vars = new List <PatternVarDef>()
{
patternVarDef
},
Node = new PatternVarRef(patternVarDef)
};
var logger = new LoggerMessageCounter();
var processor = new DslProcessor();
UstPreprocessor preprocessor = new UstPreprocessor()
{
Logger = logger
};
Expression[] resultSortedExpressions = ((PatternNode)preprocessor.Preprocess(patternVars))
.Vars.First().Values.ToArray();
Assert.AreEqual(expectedSortedExpressions.Count, resultSortedExpressions.Length);
for (int i = 0; i < expectedSortedExpressions.Count; i++)
{
Assert.IsTrue(expectedSortedExpressions[i].Equals(resultSortedExpressions[i]),
$"Not equal at {i} index: expected {expectedSortedExpressions[i]} not equals to {resultSortedExpressions[i]}");
}
}
private static void CheckJsonSerialization(string inputFileName, bool linearTextSpans = false,
bool includeTextSpans = true, bool indented = true, bool includeCode = false,
bool checkStrict = false, bool strict = true, bool checkPatternSerialization = false)
{
string path = Path.Combine(TestUtility.TestsDataPath, inputFileName);
// Serialization
string[] files = File.Exists(path)
? new string[] { path }
: Directory.GetFiles(path);
var codeRepository = new FileCodeRepository(files);
var workflow = new Workflow(codeRepository)
{
DumpStages = new HashSet <Stage>()
{
Stage.Ust
},
DumpDir = TestUtility.TestsOutputPath,
IncludeCodeInDump = includeCode,
IndentedDump = indented,
StrictJson = strict,
DumpWithTextSpans = includeTextSpans,
LineColumnTextSpans = !linearTextSpans,
Stage = Stage.Ust,
IsDumpPatterns = checkPatternSerialization
};
WorkflowResult result = workflow.Process();
CodeFile preprocessedFile = result.SourceCodeFiles.FirstOrDefault(f => f.Name == "preprocessed.php");
CodeFile originFile = result.SourceCodeFiles.FirstOrDefault(f => f.Name == "origin.php");
LineColumnTextSpan lcPreprocessedTextSpan = new LineColumnTextSpan(4, 1, 4, 3);
LineColumnTextSpan lcOriginTextSpan = new LineColumnTextSpan(3, 1, 3, 3, originFile);
var jsonFiles = new List <string>();
foreach (string file in files)
{
string shortFileName = Path.GetFileName(file) + ".ust.json";
string jsonFile = Path.Combine(TestUtility.TestsOutputPath, shortFileName);
if (file.Contains("preprocessed.php") || checkStrict)
{
string json = File.ReadAllText(jsonFile);
if (file.Contains("preprocessed.php"))
{
string preprocessedTextSpanString, originTextSpanString;
if (linearTextSpans)
{
preprocessedTextSpanString = preprocessedFile.GetTextSpan(lcPreprocessedTextSpan).ToString();
originTextSpanString = originFile.GetTextSpan(lcOriginTextSpan).ToString();
}
else
{
preprocessedTextSpanString = lcPreprocessedTextSpan.ToString();
originTextSpanString = lcOriginTextSpan.ToString();
}
json = json.Replace($"\"{preprocessedTextSpanString}\"", $"[ \"{lcPreprocessedTextSpan}\", \"{originTextSpanString}\" ]");
}
if (checkStrict)
{
json = json.Replace("\"Kind\": \"IntLiteral\"", "\"Kind\": \"IntLiteral\", \"ExcessProperty\": \"value\"");
}
File.WriteAllText(jsonFile, json);
}
jsonFiles.Add(jsonFile);
}
// Deserialization
var logger = new LoggerMessageCounter();
var newCodeRepository = new FileCodeRepository(jsonFiles)
{
LoadJson = true
};
var newPatternsRepository = checkPatternSerialization
? new JsonPatternsRepository(File.ReadAllText(Path.Combine(TestUtility.TestsOutputPath, "patterns.json")))
: inputFileName == "MultiTextSpan"
? (IPatternsRepository) new DslPatternRepository("a", "php")
: new DefaultPatternRepository();
var newWorkflow = new Workflow(newCodeRepository, newPatternsRepository)
{
StartStage = Stage.Ust,
IndentedDump = indented,
StrictJson = strict,
DumpWithTextSpans = includeTextSpans,
LineColumnTextSpans = !linearTextSpans,
Logger = logger,
};
WorkflowResult newResult = newWorkflow.Process();
if (checkStrict && strict)
{
Assert.IsTrue(logger.ErrorsString.Contains("ExcessProperty"));
}
else
{
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
Assert.GreaterOrEqual(newResult.MatchResults.Count, 1);
if (includeTextSpans)
{
if (inputFileName == "MultiTextSpan")
{
var matchResult = (MatchResult)newResult.MatchResults[1];
Assert.AreEqual(2, matchResult.TextSpans.Length);
LineColumnTextSpan actualOriginTextSpan = originFile.GetLineColumnTextSpan(matchResult.TextSpans[1]);
Assert.AreEqual(lcOriginTextSpan, actualOriginTextSpan);
LineColumnTextSpan actualPreprocessedTextSpan = preprocessedFile.GetLineColumnTextSpan(matchResult.TextSpans[0]);
Assert.AreEqual(lcPreprocessedTextSpan, actualPreprocessedTextSpan);
}
else
{
var match = (MatchResult)newResult.MatchResults.FirstOrDefault();
Assert.AreEqual(new LineColumnTextSpan(2, 1, 3, 25), result.SourceCodeFiles.First().GetLineColumnTextSpan(match.TextSpan));
}
}
}
}
