private static void YaraRulesCompileTest(FileEnumeratorParameters parameters)
{
if (!parameters.YaraParameters.Any())
{
return;
}
foreach (YaraFilter filter in parameters.YaraParameters)
{
YSScanner compiledRule = null;
try
{
compiledRule = YaraHelper.CompileRules(filter.OnMatchRules, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(YaraRulesCompileTest), string.Empty, ex);
throw;
}
finally
{
if (compiledRule != null)
{
compiledRule.Dispose();
}
}
}
}
private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters)
{
YSScanner results = null;
using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling))
{
List <YaraFilter> yaraFilters = parameters.YaraParameters;
List <string> distinctRulesToRun =
yaraFilters
.SelectMany(yf => yf.ProcessRule(this))
.Distinct()
.ToList();
if (!distinctRulesToRun.Any())
{
distinctRulesToRun =
yaraFilters
.Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
.SelectMany(yf => yf.OnMatchRules)
.Distinct()
.ToList();
}
if (!distinctRulesToRun.Any())
{
return(null);
}
distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList();
string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun);
string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken));
if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash))
{
results = _yaraCompiledRulesDictionary[ruleCollectionHash];
}
else
{
try
{
results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex);
}
_yaraCompiledRulesDictionary.Add(ruleCollectionHash, results);
}
}
return(results);
}
public static List <string> ScanFile(string filePath, YSScanner scanner)
{
List <string> result = new List <string>();
List <YSMatches> scanResults = scanner.ScanFile(filePath);
if (scanResults.Any())
{
result = scanResults.Select(match => match.Rule.Identifier).ToList();
}
return(result);
}
public static List <string> ScanBytes(byte[] fileBytes, YSScanner scanner)
{
if (fileBytes == null || fileBytes.Length == 0)
{
return(new List <string>());
}
List <string> result = new List <string>();
List <YSMatches> scanResults = new List <YSMatches>();
scanResults = scanner.ScanMemory(fileBytes);
if (scanResults.Any())
{
result = scanResults.Select(match => match.Rule.Identifier).ToList();
}
return(result);
}
public void CheckWarningsStillScan()
{
string inputFileBase = "CheckWarningStillScans";
string yaraRuleFile = Path.Combine(TestDataDirectory, $"{inputFileBase}.yar");
string yaraInputFile = Path.Combine(TestDataDirectory, $"{inputFileBase}.txt");
YSInstance yaraInstance = new YSInstance();
using (YSContext context = new YSContext())
{
using (YSCompiler compiler = new YSCompiler(null))
{
compiler.AddFile(yaraRuleFile);
YSReport compilerErrors = compiler.GetErrors();
YSReport compilerWarnings = compiler.GetWarnings();
YSScanner scanner = new YSScanner(compiler.GetRules(), null);
Assert.IsTrue(scanner.ScanFile(yaraInputFile).Any(r => r.Rule.Identifier == "WarningRule"));
}
}
}
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
_timingMetrics.Start(TimingMetric.ReadingMFTBytes);
byte[] fileBytes = new byte[0];
if (!node.Streams.Any()) //workaround for no file stream such as with hard links
{
try
{
using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read))
{
// Read the source file into a byte array.
fileBytes = new byte[fsSource.Length];
int numBytesToRead = (int)fsSource.Length;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
{
break;
}
numBytesRead += n;
numBytesToRead -= n;
}
numBytesToRead = fileBytes.Length;
}
}
catch (System.IO.IOException ioException)
{
if (ioException.Message.Contains("contains a virus"))
{
string hash = "File access blocked by anti-virus program.";
this.Sha256 = hash;
this.SHA1 = hash;
this.MD5 = hash;
return;
}
}
catch
{ }
}
else
{
fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
}
_timingMetrics.Stop(TimingMetric.ReadingMFTBytes);
_timingMetrics.Start(TimingMetric.FileHashing);
this._peData = PeDataObject.TryGetPeDataObject(fileBytes);
if (this._peData != null)
{
this.Sha256 = _peData.SHA256Hash;
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256))
{
this.Sha256 = Hash.ByteArray.Sha256(fileBytes);
this.SHA1 = Hash.ByteArray.Sha1(fileBytes);
this.MD5 = Hash.ByteArray.MD5(fileBytes);
}
_timingMetrics.Stop(TimingMetric.FileHashing);
_timingMetrics.Start(TimingMetric.MiscFileProperties);
if (_peData != null)
{
_authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
try
{
_yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex);
}
_timingMetrics.Stop(TimingMetric.YaraScanning);
}
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
_timingMetrics.Start(TimingMetric.ReadingMFTBytes);
IEnumerable <byte[]> fileChunks = node.GetBytes();
_timingMetrics.Stop(TimingMetric.ReadingMFTBytes);
_timingMetrics.Start(TimingMetric.FileHashing);
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
_timingMetrics.Stop(TimingMetric.FileHashing);
_timingMetrics.Start(TimingMetric.MiscFileProperties);
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
try
{
_yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex);
}
_timingMetrics.Stop(TimingMetric.YaraScanning);
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
