public void Match_PatternAnyWithRegex()
{
PatternRoot pattern = new PatternRoot
{
DebugInfo = "Test PatternAny",
Languages = new HashSet <Language> {
Language.Php
},
Node = new PatternAssignmentExpression
{
Left = new PatternAny("password"),
Right = new PatternStringRegexLiteral("hardcoded")
}
};
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
pattern
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
IEnumerable <MatchResultDto> matchResults = logger.Matches.ToDto();
patternsRepository.Clear();
Assert.AreEqual(1, matchResults.Count());
}
public void Process_Dsl_EqualsToHardcoded(string fileName)
{
string data = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, fileName));
var logger = new TestLogger();
var processor = new DslProcessor {
Logger = logger, PatternExpressionInsideStatement = false
};
PatternRoot result = processor.Deserialize(new TextFile(data)
{
PatternKey = fileName
});
if (fileName == "DebugInfo.pattern")
{
new HashSet <Language> {
Language.Php
};
}
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
string patternName = Path.GetFileNameWithoutExtension(fileName);
PatternRoot defaultPattern = patterns.FirstOrDefault(p => p.DebugInfo.StartsWith(patternName));
if (defaultPattern == null)
{
Assert.Inconclusive($"Pattern {patternName} does not exists in DefaultPatternRepository");
}
var patternNormalizer = new PatternNormalizer();
defaultPattern = patternNormalizer.Normalize(defaultPattern);
Assert.AreEqual(defaultPattern.Node, result.Node);
}
public void Match_Reduce_CorrectMatchingPosition()
{
var processor = new DslProcessor();
string pattern = "<[ \"\\d+\" ]>";
PatternRoot patternNode = processor.Deserialize(new TextFile(pattern)
{
PatternKey = pattern
});
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot> {
patternNode
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
List <MatchResultDto> matchResults = logger.Matches.ToDto().ToList();
patternsRepository.Clear();
Assert.AreEqual(2, matchResults.Count);
LineColumnTextSpan textSpan = matchResults[1].LineColumnTextSpan;
Assert.AreEqual(15, textSpan.BeginLine);
Assert.AreEqual(12, textSpan.BeginColumn);
Assert.AreEqual(17, textSpan.EndLine);
Assert.AreEqual(7, textSpan.EndColumn);
}
//[TestCase("#(#*, <[~e]>, #*)", new[] { 0, 1, 3 })]
public void Match_PatternExpressionsInCalls(string patternData, params int[] matchMethodNumbers)
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new TextFile(patternData)
{
PatternKey = patternData
});
patternNode.DebugInfo = patternData;
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot> {
patternNode
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
IEnumerable <MatchResultDto> matchResults = logger.Matches.ToDto();
patternsRepository.Clear();
Assert.AreEqual(matchMethodNumbers.Contains(0) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_0")));
Assert.AreEqual(matchMethodNumbers.Contains(1) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_1")));
Assert.AreEqual(matchMethodNumbers.Contains(2) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_2")));
Assert.AreEqual(matchMethodNumbers.Contains(3) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_3")));
Assert.AreEqual(matchMethodNumbers.Contains(4) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_4")));
}
public static MatchResultDto[] GetMatches(TextFile source, string pattern,
IEnumerable <Language> analyzedLanguages,
IEnumerable <Language> patternLanguages = null)
{
var sourceRep = new MemorySourceRepository(source.Data, source.FullName)
{
Languages = new HashSet <Language>(analyzedLanguages)
};
var patternsRep = new MemoryPatternsRepository();
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRep)
{
Logger = logger
};
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new TextFile(pattern)
{
PatternKey = pattern
});
patternNode.Languages = new HashSet <Language>(patternLanguages ?? LanguageUtils.PatternLanguages);
patternNode.DebugInfo = pattern;
var patternsConverter = new PatternConverter();
patternsRep.Add(patternsConverter.ConvertBack(new List <PatternRoot> {
patternNode
}));
workflow.Process();
MatchResultDto[] matchResults = logger.Matches.ToDto()
.OrderBy(r => r.PatternKey)
.ToArray();
return(matchResults);
}
public void Match_PatternVarWithRegex(string patternData)
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new TextFile(patternData)
{
PatternKey = patternData
});
patternNode.DebugInfo = patternData;
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
IEnumerable <MatchResultDto> matchResults = logger.Matches.ToDto();
patternsRepository.Clear();
int expectedMatchingCount = patternData.Contains("password") ? 1 : 0;
Assert.AreEqual(expectedMatchingCount, matchResults.Count());
}
public void Match_PatternAnyWithRegexAsSingleNode()
{
PatternRoot pattern = new PatternRoot
{
DebugInfo = "Test PatternAny",
Languages = new HashSet <Language> {
Language.Php
},
Node = new PatternAny("password")
};
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
pattern
}));
var logger = new TestLogger();
var workflow = new Workflow(sourceRep, patternsRepository)
{
Logger = logger
};
workflow.Process();
patternsRepository.Clear();
Assert.AreEqual(1, logger.Matches.Count);
Assert.AreEqual(7, ((MatchResult)logger.Matches[0]).TextSpans.Length);
}
public PatternRoot Deserialize(CodeFile data)
{
if (string.IsNullOrEmpty(data.Code))
{
throw new ParsingException(data, message: "Pattern value can not be empty.");
}
var parser = new DslAntlrParser()
{
Logger = Logger
};
var converter = new DslUstConverter
{
Logger = Logger,
PatternExpressionInsideStatement = PatternExpressionInsideStatement,
Data = data
};
DslParser.PatternContext patternContext = parser.Parse(data.Code);
PatternRoot patternNode = converter.Convert(patternContext);
patternNode.CodeFile = data;
var preprocessor = new PatternNormalizer()
{
Logger = Logger
};
patternNode = preprocessor.Normalize(patternNode);
return(patternNode);
}
public void ProcessDsl_PatternVarAlreadyDefined_HandleErrors(string data)
{
Assert.Throws(typeof(ConversionException), () =>
{
var processor = new DslProcessor();
PatternRoot result = processor.Deserialize(new CodeFile(data)
{
IsPattern = true
});
});
}
public void JsonSerialize_PatternWithVar_JsonEqualsToDsl()
{
var patternRoot = new PatternRoot
{
Node = new PatternStatements
{
Statements = new List <PatternUst>
{
new PatternAssignmentExpression
{
Left = new PatternVar("pwd")
{
Value = new PatternIdRegexToken("password")
},
Right = new PatternAny()
},
new PatternInvocationExpression
{
Target = new PatternAny(),
Arguments = new PatternArgs(
new PatternMultipleExpressions(),
new PatternVar("pwd"),
new PatternMultipleExpressions())
}
}
}
};
var jsonSerializer = new JsonPatternSerializer
{
Indented = true,
IncludeTextSpans = false
};
string json = jsonSerializer.Serialize(patternRoot);
PatternRoot nodeFromJson = jsonSerializer.Deserialize(new TextFile(json)
{
PatternKey = "PatternWithVar"
});
var dslSeializer = new DslProcessor()
{
PatternExpressionInsideStatement = false
};
var nodeFromDsl = dslSeializer.Deserialize(
new TextFile("<[@pwd:password]> = #; ... #(#*, <[@pwd]>, #*);")
{
PatternKey = "PatternWithVar2"
});
Assert.IsTrue(nodeFromJson.Node.Equals(patternRoot.Node));
Assert.IsTrue(nodeFromJson.Node.Equals(nodeFromDsl.Node));
}
public IEnumerable <PatternRoot> CreateCLangsPatterns()
{
// Init dummy pattern for correct CLangsParseTreeUst assembly initialization.
var result = new PatternRoot
{
Languages = new HashSet <Language>()
{
C.Language, CPlusPlus.Language, ObjectiveC.Language
}
};
return(Enumerable.Empty <PatternRoot>());
}
public void Parse_Dsl_WithoutErrors(string fileName)
{
string data = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, fileName));
var logger = new TestLogger();
var processor = new DslProcessor {
Logger = logger
};
PatternRoot result = processor.Deserialize(new TextFile(data)
{
PatternKey = fileName
});
Assert.AreEqual(0, logger.ErrorCount, logger.ErrorsString);
}
public void ProcessDsl_SampleWithSyntaxError_HandleErrors()
{
var logger = new LoggerMessageCounter();
var data = "(?i)password(?-i)]> = <[\"\\w*\" || null]>";
var processor = new DslProcessor()
{
Logger = logger
};
PatternRoot result = processor.Deserialize(new CodeFile(data)
{
IsPattern = true
});
Assert.AreEqual(5, logger.ErrorCount);
}
protected static void GetMatchesCount(IMatchResultBase matchResult, ref int matchedResultCount, ref int suppressedCount)
{
int patternsCount = PatternRoot.ExtractKeys(matchResult.PatternKey).Length;
if (patternsCount == 0)
{
patternsCount = 1;
}
matchedResultCount += patternsCount;
if (matchResult.Suppressed)
{
suppressedCount += patternsCount;
}
}
public void ProcessDsl_SampleWithSyntaxError_HandleErrors()
{
var logger = new TestLogger();
var data = "(?i)password(?-i)]> = <[\"\\w*\" || null]>";
var processor = new DslProcessor()
{
Logger = logger
};
PatternRoot result = processor.Deserialize(new TextFile(data)
{
PatternKey = "ErrorneousPattern"
});
Assert.AreEqual(5, logger.ErrorCount);
}
protected override List <PatternDto> InitPatterns()
{
JToken[] jsonTokens = JToken.Parse(patternsData).ReadArray();
var result = new List <PatternDto>();
JsonSerializer patternJsonSerializer = null;
foreach (JToken token in jsonTokens)
{
try
{
PatternDto patternDto;
if (token[nameof(PatternUst.Kind)] != null)
{
if (patternJsonSerializer == null)
{
patternJsonSerializer = new JsonSerializer();
var converters = patternJsonSerializer.Converters;
converters.Add(new PatternJsonConverterReader(new CodeFile(patternsData)));
var textSpanJsonConverter = new TextSpanJsonConverter();
converters.Add(textSpanJsonConverter);
converters.Add(new CodeFileJsonConverter()
{
TextSpanJsonConverter = textSpanJsonConverter
});
}
PatternRoot patternRoot = token.ToObject <PatternRoot>(patternJsonSerializer);
patternDto = patternConverter.ConvertBack(new[] { patternRoot })[0];
}
else
{
patternDto = token.ToObject <PatternDto>();
}
result.Add(patternDto);
}
catch (Exception ex)
{
Logger.LogError(ex);
}
}
return(result);
}
public PatternRoot Convert(DslParser.PatternContext pattern)
{
try
{
patternVars = new Dictionary <string, PatternVar>();
var result = new PatternRoot
{
Node = VisitPattern(pattern),
Languages = new HashSet <Language>(LanguageUtils.PatternLanguages.Values)
};
var ascendantsFiller = new PatternAscendantsFiller(result);
ascendantsFiller.FillAscendants();
return(result);
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(Data, ex));
throw;
}
}
public void Match_PatternVarWithRegex(string patternData)
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new CodeFile(patternData)
{
IsPattern = true
});
patternNode.DebugInfo = patternData;
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
WorkflowResult workflowResult = workflow.Process();
IEnumerable <MatchResultDto> matchResults = workflowResult.MatchResults.ToDto();
patternsRepository.Clear();
int expectedMatchingCount = patternData.Contains("password") ? 1 : 0;
Assert.AreEqual(expectedMatchingCount, matchResults.Count());
}
public static MatchResultDto[] GetMatches(string code, string pattern,
IEnumerable <Language> analyzedLanguages,
IEnumerable <Language> patternLanguages = null)
{
var sourceCodeRep = new MemoryCodeRepository(code)
{
Languages = new HashSet <Language>(analyzedLanguages)
};
var patternsRep = new MemoryPatternsRepository();
var workflow = new Workflow(sourceCodeRep, patternsRep)
{
Logger = new LoggerMessageCounter()
};
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new CodeFile(pattern)
{
IsPattern = true
});
patternNode.Languages = new HashSet <Language>(patternLanguages ?? LanguageUtils.PatternLanguages.Values);
patternNode.DebugInfo = pattern;
var patternsConverter = new PatternConverter();
patternsRep.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
WorkflowResult workflowResult = workflow.Process();
MatchResultDto[] matchResults = workflowResult.MatchResults.ToDto()
.OrderBy(r => r.PatternKey)
.ToArray();
return(matchResults);
}
public void Match_Reduce_CorrectMatchingPosition()
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new CodeFile("<[ \"\\d+\" ]>")
{
IsPattern = true
});
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
WorkflowResult workflowResult = workflow.Process();
List <MatchResultDto> matchResults = workflowResult.MatchResults.ToDto().ToList();
patternsRepository.Clear();
LineColumnTextSpan textSpan = matchResults[1].LineColumnTextSpan;
Assert.AreEqual(14, textSpan.BeginLine);
Assert.AreEqual(12, textSpan.BeginColumn);
Assert.AreEqual(16, textSpan.EndLine);
Assert.AreEqual(7, textSpan.EndColumn);
}
//[TestCase("#(#*, <[~e]>, #*)", new[] { 0, 1, 3 })]
public void Match_PatternExpressionsInCalls(string patternData, params int[] matchMethodNumbers)
{
var processor = new DslProcessor();
PatternRoot patternNode = processor.Deserialize(new CodeFile(patternData)
{
IsPattern = true
});
patternNode.DebugInfo = patternData;
patternsRepository.Add(patternsConverter.ConvertBack(new List <PatternRoot>()
{
patternNode
}));
WorkflowResult workflowResult = workflow.Process();
IEnumerable <MatchResultDto> matchResults = workflowResult.MatchResults.ToDto();
patternsRepository.Clear();
Assert.AreEqual(matchMethodNumbers.Contains(0) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_0")));
Assert.AreEqual(matchMethodNumbers.Contains(1) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_1")));
Assert.AreEqual(matchMethodNumbers.Contains(2) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_2")));
Assert.AreEqual(matchMethodNumbers.Contains(3) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_3")));
Assert.AreEqual(matchMethodNumbers.Contains(4) ? 1 : 0, matchResults.Count(r => r.MatchedCode.StartsWith("test_call_4")));
}
public string Serialize(PatternRoot patternRoot)
{
return(patternRoot.Node.ToString());
}
public static IPatternsRepository CreatePatternsRepository(string patternsString,
IEnumerable <string> patternIds,
ILogger logger)
{
IPatternsRepository patternsRepository;
if (string.IsNullOrEmpty(patternsString) || patternsString == "default")
{
patternsRepository = new DefaultPatternRepository();
}
else if (patternsString.EqualsIgnoreCase("no"))
{
patternsRepository = DummyPatternsRepository.Instance;
}
else if (patternsString.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
{
patternsRepository = new JsonPatternsRepository(FileExt.ReadAllText(patternsString));
}
else
{
TextFile patternsFile;
if (patternsString.EndsWith(".pattern", StringComparison.OrdinalIgnoreCase))
{
patternsFile = new TextFile(FileExt.ReadAllText(patternsString))
{
PatternKey = patternsString,
Name = patternsString
};
}
else
{
patternsFile = new TextFile(patternsString);
}
var processor = new DslProcessor();
if (logger != null)
{
processor.Logger = logger;
}
PatternRoot patternRoot = processor.Deserialize(patternsFile);
var patternConverter = new PatternConverter();
if (logger != null)
{
patternConverter.Logger = logger;
}
List <PatternDto> dtos = patternConverter.ConvertBack(new[] { patternRoot });
var memoryPatternsRepository = new MemoryPatternsRepository();
memoryPatternsRepository.Add(dtos);
patternsRepository = memoryPatternsRepository;
}
if (logger != null)
{
patternsRepository.Logger = logger;
}
patternsRepository.Identifiers = patternIds as List <string> ?? patternIds?.ToList();
return(patternsRepository);
}
private PatternDto ProcessToken(JToken token)
{
try
{
PatternDto patternDto;
if (token[nameof(PatternUst.Kind)] != null)
{
if (patternJsonSerializer == null)
{
patternJsonSerializer = new JsonSerializer();
var converters = patternJsonSerializer.Converters;
var patternJsonConverterReader = new PatternJsonConverterReader(new TextFile(patternsData))
{
Logger = Logger,
DefaultDataFormat = DefaultDataFormat,
DefaultKey = DefaultKey,
DefaultFilenameWildcard = DefaultFilenameWildcard,
DefaultLanguages = DefaultLanguages
};
converters.Add(patternJsonConverterReader);
var textSpanJsonConverter = new TextSpanJsonConverter
{
Logger = Logger
};
converters.Add(textSpanJsonConverter);
var sourceFileJsonConverter = new SourceFileJsonConverter
{
Logger = Logger,
SetCurrentSourceFileAction = sourceFile =>
{
textSpanJsonConverter.CurrentSourceFile = sourceFile;
}
};
converters.Add(sourceFileJsonConverter);
}
PatternRoot patternRoot = token.ToObject <PatternRoot>(patternJsonSerializer);
patternDto = patternConverter.ConvertBack(new[] { patternRoot })[0];
}
else
{
patternDto = new PatternDto
{
Name = token[nameof(PatternDto.Name)]?.ToString() ?? string.Empty,
Key = token[nameof(PatternDto.Key)]?.ToString() ?? string.Empty,
DataFormat = token[nameof(PatternDto.DataFormat)]?.ToString(),
Value = token[nameof(PatternDto.Value)]?.ToString(),
CweId = token[nameof(PatternDto.CweId)]?.ToString(),
Description = token[nameof(PatternDto.Description)]?.ToString(),
FilenameWildcard = token[nameof(PatternDto.FilenameWildcard)]?.ToString()
};
HashSet <string> languages;
var languagesToken = token["Languages"];
if (languagesToken == null)
{
languages = new HashSet <string>();
}
else if (languagesToken is JArray)
{
languages = new HashSet <string>(languagesToken.Select(x => x.ToString()));
}
else
{
languages = new HashSet <string>(languagesToken.ToString()
.Split(',').Select(x => x.Trim()));
}
patternDto.Languages = languages;
}
return(patternDto);
}
catch (Exception ex)
{
Logger.LogError(ex);
}
return(null);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return(null);
}
JObject jObject = JObject.Load(reader);
object target = null;
if (objectType == typeof(PatternRoot))
{
HashSet <Language> resultLanguages = ((string)jObject[nameof(PatternDto.Languages)])?.ParseLanguages(patternLanguages: true);
root = new PatternRoot
{
Key = (string)jObject[nameof(PatternRoot.Key)] ?? "",
FilenameWildcard = (string)jObject[nameof(PatternRoot.FilenameWildcard)] ?? "",
Languages = resultLanguages,
DataFormat = (string)jObject[nameof(PatternRoot.DataFormat)] ?? "",
CodeFile = jObject[nameof(PatternRoot.CodeFile)]?.ToObject <CodeFile>(serializer) ?? CodeFile.Empty,
};
target = root;
root.Node = jObject[nameof(PatternRoot.Node)].ToObject <PatternUst>(serializer);
}
else if (objectType == typeof(PatternUst) || objectType.IsSubclassOf(typeof(PatternUst)))
{
var kind = (string)jObject[nameof(PatternUst.Kind)];
ReflectionCache.TryGetClassType(kind, out Type type);
var patternUst = (PatternUst)Activator.CreateInstance(type);
target = patternUst;
patternUst.Root = root;
if (ancestors.Count > 0)
{
patternUst.Parent = ancestors.Peek();
}
ancestors.Push(patternUst);
if (patternUst is IRegexPattern regexPattern)
{
if ((string)jObject[nameof(IRegexPattern.Regex)] is string regex)
{
regexPattern.RegexString = regex;
}
ReadTextSpan(jObject, patternUst);
}
else if (patternUst is PatternIntRangeLiteral patternIntRangeLiteral)
{
if ((string)jObject[nameof(PatternIntLiteral.Value)] is string range)
{
patternIntRangeLiteral.ParseAndPopulate(range);
}
ReadTextSpan(jObject, patternUst);
}
else
{
serializer.Populate(jObject.CreateReader(), target);
}
ancestors.Pop();
}
return(target);
}
