public void Match_Reduce_CorrectMatchingPosition()
{
var processor = new DslProcessor();
string pattern = "<[ \"\\d+\" ]>";
PatternRoot patternNode = processor.Deserialize(new TextFile(pattern)
{
PatternKey = pattern
});
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot> {
patternNode
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
List <MatchResultDto> matchResults = logger.Matches.ToDto().ToList();
patternsRepository.Clear();
Assert.AreEqual(2, matchResults.Count);
LineColumnTextSpan textSpan = matchResults[1].LineColumnTextSpan;
Assert.AreEqual(15, textSpan.BeginLine);
Assert.AreEqual(12, textSpan.BeginColumn);
Assert.AreEqual(17, textSpan.EndLine);
Assert.AreEqual(7, textSpan.EndColumn);
}
public void Parse_LineColumnTextSpan(int beginLine, int beginCol, int endLine, int endCol)
{
var lcTextSpan = new LineColumnTextSpan(beginLine, beginCol, endLine, endCol);
string textSpanString = lcTextSpan.ToString();
Assert.AreEqual(lcTextSpan, TextUtils.ParseLineColumnTextSpan(textSpanString));
}
public static bool IsSuppressed(TextFile sourceFile, LineColumnTextSpan lineColumnTextSpan)
{
string prevLine = lineColumnTextSpan.BeginLine - 1 > 0
? sourceFile.GetStringAtLine(lineColumnTextSpan.BeginLine - 1)
: "";
return(SupressMarkerRegex.IsMatch(prevLine));
}
public TextSpan GetTextSpan(LineColumnTextSpan textSpan)
{
int start = GetLinearFromLineColumn(textSpan.BeginLine, textSpan.BeginColumn);
int end = GetLinearFromLineColumn(textSpan.EndLine, textSpan.EndColumn);
var result = TextSpan.FromBounds(start, end, textSpan.File);
return(result);
}
static void Main(string[] args)
{
var fileRepository = new FileCodeRepository("code.php");
var patternsRepostiory = new FilePatternsRepository("patterns.json");
var workflow = new Workflow(fileRepository, patternsRepostiory);
workflow.IsIncludeIntermediateResult = true;
WorkflowResult result = workflow.Process();
var lineTokenDict = new Dictionary <int, List <string> >();
foreach (var matchingResult in result.MatchingResults)
{
var matchedNode = matchingResult.Nodes.Last();
UstNode[] descendants = matchedNode.GetAllDescendants();
foreach (UstNode descendant in descendants)
{
string text = null;
if (descendant.NodeType == NodeType.InvocationExpression)
{
text = "Method invocation: " + ((InvocationExpression)descendant).Target;
}
else if (descendant.NodeType == NodeType.IndexerDeclaration)
{
text = "Arrays with user input: " + ((IndexerExpression)descendant).Target;
}
else if (descendant.NodeType == NodeType.CastExpression)
{
text = "Cast to type: " + ((CastExpression)descendant).Type;
}
if (!string.IsNullOrEmpty(text))
{
SourceCodeFile sourceCodeFile = result.SourceCodeFiles.Single(file => file.FullPath == descendant.FileNode.FileName.Text);
LineColumnTextSpan lineColumnTextSpan = sourceCodeFile.GetLineColumnTextSpan(descendant.TextSpan);
List <string> strs;
if (!lineTokenDict.TryGetValue(lineColumnTextSpan.BeginLine, out strs))
{
strs = new List <string>();
lineTokenDict[lineColumnTextSpan.BeginLine] = strs;
}
strs.Add(text);
}
}
}
foreach (var item in lineTokenDict)
{
Console.WriteLine(String.Format("{0}: {1}", item.Key, string.Join(", ", item.Value)));
}
Console.ReadLine();
}
public void Parse_NewLine_CorrectLineColumn()
{
string fileText = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, "newLine -r-n.php"));
var lineEnds = new [] { "\r", "\n", "\r\n" };
foreach (var lineEnd in lineEnds)
{
var phpParser = new PhpAntlrParser();
string code = fileText.Replace("\r\n", lineEnd);
var sourceFile = new TextFile(code)
{
Name = "newLine.php",
};
var tokens = Language.Php.CreateAntlrLexer().GetTokens(sourceFile, out TimeSpan _);
phpParser.SourceFile = sourceFile;
var parseTree = (PhpAntlrParseTree)phpParser.Parse(tokens, out TimeSpan _);
var converter = new PhpAntlrParseTreeConverter();
RootUst ust = converter.Convert(parseTree);
Ust intNode = ust.WhereDescendantsOrSelf(
node => node is IntLiteral intLiteral && intLiteral.Value == 42).First();
LineColumnTextSpan intNodeSpan = intNode.LineColumnTextSpan;
Assert.AreEqual(1, intNodeSpan.BeginLine);
Assert.AreEqual(12, intNodeSpan.BeginColumn);
Assert.AreEqual(14, intNodeSpan.EndColumn);
Ust heredocNode = ust.WhereDescendantsOrSelf(
node => node is StringLiteral stringLiteral &&
stringLiteral.Text.StartsWith("Heredoc text")).First();
LineColumnTextSpan heredocNodeSpan = heredocNode.LineColumnTextSpan;
Assert.AreEqual(3, heredocNodeSpan.BeginLine);
Assert.AreEqual(6, heredocNodeSpan.EndLine);
Ust serverAddressNode = ust.WhereDescendantsOrSelf(
node => node is StringLiteral stringLiteral &&
stringLiteral.Text.Contains("http://127.0.0.1")).First();
LineColumnTextSpan serverAddressNodeSpan = serverAddressNode.LineColumnTextSpan;
Assert.AreEqual(8, serverAddressNodeSpan.BeginLine);
Assert.AreEqual(15, serverAddressNodeSpan.BeginColumn);
}
}
private TextSpan DeserializeTextSpan(JsonReader reader, bool isLineColumn)
{
TextSpan result = TextSpan.Zero;
string textSpanString = (string)reader.Value;
if (textSpanString != EmptyTextSpanFormat)
{
if (!isLineColumn)
{
try
{
result = TextUtils.ParseTextSpan(textSpanString, CurrentCodeFile, CodeFiles);
}
catch (FileNotFoundException ex)
{
Logger.LogError(JsonFile, (reader as JTokenReader)?.CurrentToken, ex);
}
}
else
{
try
{
LineColumnTextSpan lineColumnTextSpan = TextUtils.ParseLineColumnTextSpan(textSpanString, CurrentCodeFile, CodeFiles);
result = lineColumnTextSpan.CodeFile.GetTextSpan(lineColumnTextSpan);
}
catch (FileNotFoundException ex)
{
Logger.LogError(JsonFile, (reader as JTokenReader)?.CurrentToken, ex);
}
}
if (result.CodeFile == CurrentCodeFile)
{
result.CodeFile = null;
}
}
return(result);
}
public void Match_Comments_CorrectMatchingPosition()
{
TextFile source = new TextFile(
"<?php\n" +
"#password=secret\n" +
"/*password=secret*/\n" +
"/*\n" +
"\n" +
" password\n" +
" =secret\n" +
"*/" +
"?>",
"code.php");
var pattern = "Comment: <[ \"(?i)(password|pwd)\\s*(\\=|is|\\:)\" ]>";
MatchResultDto[] matchResults = PatternMatchingUtils.GetMatches(source, pattern, Language.Php);
LineColumnTextSpan textSpan0 = matchResults[0].LineColumnTextSpan;
Assert.AreEqual(2, textSpan0.BeginLine);
Assert.AreEqual(2, textSpan0.BeginColumn);
Assert.AreEqual(2, textSpan0.EndLine);
Assert.AreEqual(11, textSpan0.EndColumn);
LineColumnTextSpan textSpan1 = matchResults[1].LineColumnTextSpan;
Assert.AreEqual(3, textSpan1.BeginLine);
Assert.AreEqual(3, textSpan1.BeginColumn);
Assert.AreEqual(3, textSpan1.EndLine);
Assert.AreEqual(12, textSpan1.EndColumn);
LineColumnTextSpan textSpan2 = matchResults[2].LineColumnTextSpan;
Assert.AreEqual(6, textSpan2.BeginLine);
Assert.AreEqual(5, textSpan2.BeginColumn);
Assert.AreEqual(7, textSpan2.EndLine);
Assert.AreEqual(16, textSpan2.EndColumn);
}
public static void LogConversionError(this ILogger logger, Exception ex,
ParserRuleContext context, CodeFile currentFileData)
{
StackTrace stackTrace = new StackTrace(ex, true);
int frameNumber = 0;
string fileName = null;
string methodName = null;
int line = 0;
int column = 0;
do
{
StackFrame frame = stackTrace.GetFrame(frameNumber);
fileName = frame.GetFileName();
methodName = frame.GetMethod().Name;
line = frame.GetFileLineNumber();
column = frame.GetFileColumnNumber();
frameNumber++;
}while (frameNumber < stackTrace.FrameCount && (fileName == null || methodName == "Visit"));
var textSpan = context.GetTextSpan();
string exceptionText;
LineColumnTextSpan lineColumnTextSpan = currentFileData.GetLineColumnTextSpan(textSpan);
if (fileName != null)
{
exceptionText = $"{ex.Message} at method \"{methodName}\" {line}:{column} at position {lineColumnTextSpan.BeginLine}:{lineColumnTextSpan.BeginColumn} in source file";
}
else
{
exceptionText = $"{ex.Message} at position {lineColumnTextSpan.BeginLine}:{lineColumnTextSpan.BeginColumn} in source file";
}
logger.LogError(new ConversionException(currentFileData, message: exceptionText)
{
TextSpan = textSpan
});
}
public PatternUst VisitPatternLiterals([NotNull] DslParser.PatternLiteralsContext context)
{
IEnumerable <PatternUst> values = context.patternNotLiteral()
.Select(literal => VisitPatternNotLiteral(literal));
var patternOr = values.Count() > 1
? new PatternOr(values)
: values.Count() == 1
? values.First()
: new PatternIdRegexToken();
PatternUst result;
if (context.PatternVar() != null)
{
string id = context.PatternVar().GetText().Substring(1);
if (values.Count() > 0 && patternVars.TryGetValue(id, out PatternVar existedPatternVar))
{
LineColumnTextSpan lcTextSpan = Data.GetLineColumnTextSpan(existedPatternVar.TextSpan);
throw new ConversionException(
Data,
message: $"DSL Error: PatternVar {id} with the same Id already defined earlier at {lcTextSpan}")
{
TextSpan = context.PatternVar().GetTextSpan()
};
}
var patternVar = new PatternVar(id, context.GetTextSpan());
patternVars[id] = patternVar;
patternVar.Value = patternOr;
result = patternVar;
}
else
{
result = patternOr;
}
return(result);
}
public void Match_Reduce_CorrectMatchingPosition()
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new CodeFile("<[ \"\\d+\" ]>")
{
IsPattern = true
});
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
WorkflowResult workflowResult = workflow.Process();
List <MatchResultDto> matchResults = workflowResult.MatchResults.ToDto().ToList();
patternsRepository.Clear();
LineColumnTextSpan textSpan = matchResults[1].LineColumnTextSpan;
Assert.AreEqual(14, textSpan.BeginLine);
Assert.AreEqual(12, textSpan.BeginColumn);
Assert.AreEqual(16, textSpan.EndLine);
Assert.AreEqual(7, textSpan.EndColumn);
}
public UstNode VisitPatternLiterals([NotNull] DslParser.PatternLiteralsContext context)
{
Expression result;
PatternVarDef patternVarDef;
if (context.patternNotLiteral().Length == 1)
{
result = (Expression)VisitPatternNotLiteral(context.patternNotLiteral().First());
if (context.PatternVar() != null)
{
string id = context.PatternVar().GetText().Substring(1);
if (!patternVarDefs.TryGetValue(id, out patternVarDef))
{
patternVarDef = new PatternVarDef(id, new Expression[] { result }, context.GetTextSpan());
patternVarDefs[id] = patternVarDef;
}
else
{
if (context.patternNotLiteral().Length != 0)
{
var lcTextSpan = new LineColumnTextSpan(patternVarDef.TextSpan, Data);
throw new ConversionException(
$"DSL Error: PatternVar {id} with matching Id already defined earlier at {lcTextSpan}")
{
TextSpan = context.PatternVar().GetTextSpan()
};
}
}
result = new PatternVarRef(patternVarDef, context.GetTextSpan());
}
}
else
{
List <Expression> values = context.patternNotLiteral()
.Select(literal => (Expression)VisitPatternNotLiteral(literal)).ToList();
if (values.Count == 0)
{
values.Add(new PatternIdToken("", context.GetTextSpan()));
}
if (context.PatternVar() == null)
{
result = new PatternVarDef(GetNewVarDefName(), values, context.GetTextSpan());
}
else
{
string id = context.PatternVar().GetText().Substring(1);
if (!patternVarDefs.TryGetValue(id, out patternVarDef))
{
patternVarDef = new PatternVarDef(id, values, context.GetTextSpan());
patternVarDefs[id] = patternVarDef;
}
else
{
if (context.patternNotLiteral().Length != 0)
{
var lcTextSpan = new LineColumnTextSpan(patternVarDef.TextSpan, Data);
throw new ConversionException(
$"DSL Error: PatternVar {id} with matching Id already defined earlier at {lcTextSpan}")
{
TextSpan = context.PatternVar().GetTextSpan()
};
}
}
result = new PatternVarRef(patternVarDef, context.GetTextSpan());
}
}
return(result);
}
private static void CheckJsonSerialization(string inputFileName, bool linearTextSpans = false,
bool includeTextSpans = true, bool indented = true, bool includeCode = false,
bool checkStrict = false, bool strict = true, bool checkPatternSerialization = false)
{
string path = Path.Combine(TestUtility.TestsDataPath, inputFileName);
// Serialization
string[] files = File.Exists(path)
? new [] { path }
: Directory.GetFiles(path);
var codeRepository = new FileSourceRepository(files);
var logger = new TestLogger();
var workflow = new Workflow(codeRepository)
{
DumpStages = new HashSet <Stage> {
Stage.Ust
},
DumpDir = TestUtility.TestsOutputPath,
IncludeCodeInDump = includeCode,
IndentedDump = indented,
StrictJson = strict,
DumpWithTextSpans = includeTextSpans,
LineColumnTextSpans = !linearTextSpans,
Stage = Stage.Ust,
IsDumpPatterns = checkPatternSerialization
};
WorkflowResult result = workflow.Process();
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
var preprocessedFile = (TextFile)result.SourceFiles.FirstOrDefault(f => f.Name == "preprocessed.php");
var originFile = (TextFile)result.SourceFiles.FirstOrDefault(f => f.Name == "origin.php");
LineColumnTextSpan lcPreprocessedTextSpan = new LineColumnTextSpan(4, 1, 4, 3);
LineColumnTextSpan lcOriginTextSpan = new LineColumnTextSpan(3, 1, 3, 3, originFile);
var jsonFiles = new List <string>();
foreach (string file in files)
{
string shortFileName = Path.GetFileName(file) + ".ust.json";
string jsonFile = Path.Combine(TestUtility.TestsOutputPath, shortFileName);
if (file.Contains("preprocessed.php") || checkStrict)
{
string json = File.ReadAllText(jsonFile);
if (file.Contains("preprocessed.php"))
{
string preprocessedTextSpanString, originTextSpanString;
if (linearTextSpans)
{
preprocessedTextSpanString = preprocessedFile.GetTextSpan(lcPreprocessedTextSpan).ToString();
originTextSpanString = originFile.GetTextSpan(lcOriginTextSpan).ToString();
}
else
{
preprocessedTextSpanString = lcPreprocessedTextSpan.ToString();
originTextSpanString = lcOriginTextSpan.ToString();
}
json = json.Replace($"\"{preprocessedTextSpanString}\"", $"[ \"{lcPreprocessedTextSpan}\", \"{originTextSpanString}\" ]");
}
if (checkStrict)
{
json = json.Replace("\"Kind\": \"IntLiteral\"", "\"Kind\": \"IntLiteral\", \"ExcessProperty\": \"value\"");
}
File.WriteAllText(jsonFile, json);
}
jsonFiles.Add(jsonFile);
}
// Deserialization
var newLogger = new TestLogger();
var newCodeRepository = new FileSourceRepository(jsonFiles);
var newPatternsRepository = checkPatternSerialization
? new JsonPatternsRepository(File.ReadAllText(Path.Combine(TestUtility.TestsOutputPath, "patterns.json")))
: inputFileName == "MultiTextSpan"
? (IPatternsRepository) new DslPatternRepository("a", "php")
: new DefaultPatternRepository();
var newWorkflow = new Workflow(newCodeRepository, newPatternsRepository)
{
StrictJson = strict,
Logger = newLogger
};
newWorkflow.Process();
if (checkStrict && strict)
{
Assert.IsTrue(newLogger.ErrorsString.Contains("ExcessProperty"));
}
else
{
Assert.AreEqual(0, newLogger.ErrorCount, newLogger.ErrorsString);
Assert.GreaterOrEqual(newLogger.Matches.Count, 1);
if (includeTextSpans)
{
if (inputFileName == "MultiTextSpan")
{
var matchResult = (MatchResult)newLogger.Matches[1];
Assert.AreEqual(2, matchResult.TextSpans.Length);
LineColumnTextSpan actualOriginTextSpan = originFile.GetLineColumnTextSpan(matchResult.TextSpans[1]);
Assert.AreEqual(lcOriginTextSpan, actualOriginTextSpan);
LineColumnTextSpan actualPreprocessedTextSpan = preprocessedFile.GetLineColumnTextSpan(matchResult.TextSpans[0]);
Assert.AreEqual(lcPreprocessedTextSpan, actualPreprocessedTextSpan);
}
else
{
var match = (MatchResult)newLogger.Matches[0];
var enumerator = result.SourceFiles.GetEnumerator();
enumerator.MoveNext();
var firstFile = (TextFile)enumerator.Current;
Assert.AreEqual(new LineColumnTextSpan(2, 1, 3, 25), firstFile.GetLineColumnTextSpan(match.TextSpan));
}
}
}
}
