private static void YaraRulesCompileTest(FileEnumeratorParameters parameters)
{
if (!parameters.YaraParameters.Any())
{
return;
}
foreach (YaraFilter filter in parameters.YaraParameters)
{
YSScanner compiledRule = null;
try
{
compiledRule = YaraHelper.CompileRules(filter.OnMatchRules, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(YaraRulesCompileTest), string.Empty, ex);
throw;
}
finally
{
if (compiledRule != null)
{
compiledRule.Dispose();
}
}
}
}
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
IEnumerable <byte[]> fileChunks = node.GetBytes();
_timingMetrics.Start(TimingMetric.FileHashing);
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
_timingMetrics.Stop(TimingMetric.FileHashing);
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
_timingMetrics.Stop(TimingMetric.YaraScanning);
compiledRules = null;
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters)
{
YSScanner results = null;
using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling))
{
List <YaraFilter> yaraFilters = parameters.YaraParameters;
List <string> distinctRulesToRun =
yaraFilters
.SelectMany(yf => yf.ProcessRule(this))
.Distinct()
.ToList();
if (!distinctRulesToRun.Any())
{
distinctRulesToRun =
yaraFilters
.Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
.SelectMany(yf => yf.OnMatchRules)
.Distinct()
.ToList();
}
if (!distinctRulesToRun.Any())
{
return(null);
}
distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList();
string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun);
string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken));
if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash))
{
results = _yaraCompiledRulesDictionary[ruleCollectionHash];
}
else
{
try
{
results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex);
}
_yaraCompiledRulesDictionary.Add(ruleCollectionHash, results);
}
}
return(results);
}
public void UpdateExistingYaraRule(SqlKey key, List <string> newYaraMatchedRules)
{
string newYaraRulesMatchedValue = YaraHelper.FormatDelimitedRulesString(newYaraMatchedRules);
SQLiteParameter yaraMatchedRulesParameter = SqlHelper.GetParameter("YaraRulesMatched", newYaraRulesMatchedValue);
List <SQLiteParameter> parameters = key.GetSqlParameters().ToList();
parameters.Add(yaraMatchedRulesParameter);
string commandText = string.Format(SqlStrings.UpdateYaraRules, yaraMatchedRulesParameter.ParameterName);
ExecuteNonQuery(commandText, parameters);
}
private bool InsertIntoDB(FileProperties fileProperties, SqlKey key, List <SqlParameter> sqlParameters)
{
string updateText = string.Empty;
if (!string.IsNullOrWhiteSpace(fileProperties.YaraMatchedRules))
{
List <string> newYaraMatchedRules = new List <string>();
string currentYaraRulesMatchedValue = GetExistingYaraRules(key);
if (currentYaraRulesMatchedValue != null)
{
newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(currentYaraRulesMatchedValue));
}
newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(fileProperties.YaraMatchedRules));
string newYaraRulesMatchedValue = YaraHelper.FormatDelimitedRulesString(newYaraMatchedRules);
SqlParameter yaraMatchedValueParameter = ParameterHelper.GetNewStringParameter("YaraRulesMatched", newYaraRulesMatchedValue);
sqlParameters.Add(yaraMatchedValueParameter);
updateText = $"UPDATE [{TableName}] SET [YaraRulesMatched] = {yaraMatchedValueParameter.ParameterName} WHERE {key.GetWhereClause()}";
}
string columnNames = sqlParameters.AsColumnString();
string values = sqlParameters.AsValuesString();
string insertStatement =
$@" INSERT INTO [{TableName}]
({columnNames},[PrevalenceCount],[DateSeen])
VALUES
({values},1,GETDATE())" ;
string commandText =
$@"DECLARE @PREVALENCECOUNT INT;
SET @PREVALENCECOUNT = ( SELECT [PrevalenceCount] FROM [{TableName}] WHERE {key.GetWhereClause()} )
SET @PREVALENCECOUNT = @PREVALENCECOUNT + 1;
IF(@PREVALENCECOUNT IS NOT NULL)
BEGIN
UPDATE [{TableName}] SET [PrevalenceCount] = @PREVALENCECOUNT WHERE {key.GetWhereClause()}
{updateText}
END
ELSE
BEGIN
{insertStatement}
END
";
return(ExecNonQuery(commandText, sqlParameters));
}
public static void CleanUp()
{
if (_yaraCompiledRulesDictionary.Any())
{
List <YSScanner> rules = _yaraCompiledRulesDictionary.Values.ToList();
foreach (YSScanner scanner in rules)
{
scanner.Dispose();
}
_yaraCompiledRulesDictionary.Clear();
_yaraCompiledRulesDictionary = null;
}
YaraHelper.CleanUp();
}
private string PopulateYaraInfo(List <YaraFilter> yaraFilters)
{
List <string> distinctRulesToRun =
yaraFilters
.SelectMany(yf => yf.ProcessRule(this))
.Distinct()
.OrderBy(s => s)
.ToList();
if (!distinctRulesToRun.Any())
{
distinctRulesToRun =
yaraFilters
.Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
.SelectMany(yf => yf.OnMatchRules)
.Distinct()
.ToList();
}
if (!distinctRulesToRun.Any())
{
return(string.Empty);
}
_timingMetrics.Start(TimingMetric.YaraIndexBuilding);
string yaraIndexContents = YaraHelper.MakeYaraIndexFile(distinctRulesToRun);
string indexFileHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(yaraIndexContents));
string yaraIndexFilename = Path.Combine(Path.GetTempPath(), $"{indexFileHash}-index.yar");
if (!File.Exists(yaraIndexFilename))
{
File.WriteAllText(yaraIndexFilename, yaraIndexContents);
}
_timingMetrics.Stop(TimingMetric.YaraIndexBuilding);
return(yaraIndexFilename);
}
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
_timingMetrics.Start(TimingMetric.ReadingMFTBytes);
byte[] fileBytes = new byte[0];
if (!node.Streams.Any()) //workaround for no file stream such as with hard links
{
try
{
using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read))
{
// Read the source file into a byte array.
fileBytes = new byte[fsSource.Length];
int numBytesToRead = (int)fsSource.Length;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
{
break;
}
numBytesRead += n;
numBytesToRead -= n;
}
numBytesToRead = fileBytes.Length;
}
}
catch (System.IO.IOException ioException)
{
if (ioException.Message.Contains("contains a virus"))
{
string hash = "File access blocked by anti-virus program.";
this.Sha256 = hash;
this.SHA1 = hash;
this.MD5 = hash;
return;
}
}
catch
{ }
}
else
{
fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
}
_timingMetrics.Stop(TimingMetric.ReadingMFTBytes);
_timingMetrics.Start(TimingMetric.FileHashing);
this._peData = PeDataObject.TryGetPeDataObject(fileBytes);
if (this._peData != null)
{
this.Sha256 = _peData.SHA256Hash;
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256))
{
this.Sha256 = Hash.ByteArray.Sha256(fileBytes);
this.SHA1 = Hash.ByteArray.Sha1(fileBytes);
this.MD5 = Hash.ByteArray.MD5(fileBytes);
}
_timingMetrics.Stop(TimingMetric.FileHashing);
_timingMetrics.Start(TimingMetric.MiscFileProperties);
if (_peData != null)
{
_authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
try
{
_yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex);
}
_timingMetrics.Stop(TimingMetric.YaraScanning);
}
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
IEnumerable <byte[]> fileChunks = null;
using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes))
{
fileChunks = node.GetBytes();
}
using (var timer = new TimingMetrics(TimingMetric.FileHashing))
{
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
}
using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties))
{
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
}
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
using (var timer = new TimingMetrics(TimingMetric.YaraScanning))
{
try
{
_yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex);
}
}
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy))
{
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
}
CancellationHelper.ThrowIfCancelled();
}
}
public bool PersistFileProperties(FileProperties fileProperties)
{
SqlKey key = new SqlKey(fileProperties.MFTNumber, fileProperties.SequenceNumber, fileProperties.Sha256);
List <SQLiteParameter> sqlParameters = new List <SQLiteParameter>();
sqlParameters.AddRange(key.GetSqlParameters());
sqlParameters.AddRange(new List <SQLiteParameter>
{
SqlHelper.GetParameter("DriveLetter", fileProperties.DriveLetter),
SqlHelper.GetParameter("FullPath", fileProperties.FullPath),
SqlHelper.GetParameter("Filename", fileProperties.FileName),
SqlHelper.GetParameter("Extension", fileProperties.Extension),
SqlHelper.GetParameter("DirectoryLocation", fileProperties.DirectoryLocation),
SqlHelper.GetParameter("Length", fileProperties.Length),
SqlHelper.GetParameter("MftTimeCreation", fileProperties.MftTimeCreation),
SqlHelper.GetParameter("MftTimeAccessed", fileProperties.MftTimeAccessed),
SqlHelper.GetParameter("MftTimeModified", fileProperties.MftTimeModified),
SqlHelper.GetParameter("MftTimeMftModified", fileProperties.MftTimeMftModified),
SqlHelper.GetParameter("CreationTime", fileProperties.CreationTime),
SqlHelper.GetParameter("LastAccessTime", fileProperties.LastAccessTime),
SqlHelper.GetParameter("LastWriteTime", fileProperties.LastWriteTime),
SqlHelper.GetParameter("Project", fileProperties.Project),
SqlHelper.GetParameter("ProviderItemID", fileProperties.ProviderItemID),
SqlHelper.GetParameter("OriginalFileName", fileProperties.OriginalFileName),
SqlHelper.GetParameter("FileOwner", fileProperties.FileOwner),
SqlHelper.GetParameter("FileVersion", fileProperties.FileVersion),
SqlHelper.GetParameter("FileDescription", fileProperties.FileDescription),
SqlHelper.GetParameter("Trademarks", fileProperties.Trademarks),
SqlHelper.GetParameter("Copyright", fileProperties.Copyright),
SqlHelper.GetParameter("Company", fileProperties.Company),
SqlHelper.GetParameter("ApplicationName", fileProperties.ApplicationName),
SqlHelper.GetParameter("Comment", fileProperties.Comment),
SqlHelper.GetParameter("Title", fileProperties.Title),
SqlHelper.GetParameter("Link", fileProperties.Link),
SqlHelper.GetParameter("MimeType", fileProperties.MimeType),
SqlHelper.GetParameter("InternalName", fileProperties.InternalName),
SqlHelper.GetParameter("ProductName", fileProperties.ProductName),
SqlHelper.GetParameter("Language", fileProperties.Language),
SqlHelper.GetParameter("ComputerName", fileProperties.ComputerName),
SqlHelper.GetParameter("Attributes", fileProperties.Attributes),
SqlHelper.GetParameter("SHA1", fileProperties.SHA1),
SqlHelper.GetParameter("MD5", fileProperties.MD5),
SqlHelper.GetParameter("ImpHash", fileProperties.ImpHash),
SqlHelper.GetParameter("IsDll", fileProperties.IsDll),
SqlHelper.GetParameter("IsExe", fileProperties.IsExe),
SqlHelper.GetParameter("IsDriver", fileProperties.IsDriver),
SqlHelper.GetParameter("IsSigned", fileProperties.IsSigned),
SqlHelper.GetParameter("IsSignatureValid", fileProperties.IsSignatureValid),
SqlHelper.GetParameter("IsValidCertChain", fileProperties.IsValidCertChain),
SqlHelper.GetNewParameterByType("BinaryType", fileProperties.BinaryType.GetValueOrDefault(), DbType.Int32),
SqlHelper.GetNewParameterByType("CompileDate", fileProperties.CompileDate.GetValueOrDefault(), DbType.DateTime2),
SqlHelper.GetParameter("IsTrusted", fileProperties.IsTrusted),
SqlHelper.GetParameter("CertSubject", fileProperties.CertSubject),
SqlHelper.GetParameter("CertIssuer", fileProperties.CertIssuer),
SqlHelper.GetParameter("CertSerialNumber", fileProperties.CertSerialNumber),
SqlHelper.GetParameter("CertThumbprint", fileProperties.CertThumbprint),
SqlHelper.GetParameter("CertNotBefore", fileProperties.CertNotBefore),
SqlHelper.GetParameter("CertNotAfter", fileProperties.CertNotAfter),
SqlHelper.GetParameter("Entropy", fileProperties.Entropy ?? 0)
});
int count = _dataClient.GetPrevalenceCount(key);
if (count == -1)
{
_dataClient.InsertRow(sqlParameters);
return(true);
}
count += 1;
if (!string.IsNullOrWhiteSpace(fileProperties.YaraMatchedRules))
{
List <string> newYaraMatchedRules = new List <string>();
string currentYaraRulesMatchedValue = _dataClient.GetExistingYaraRules(key);
if (currentYaraRulesMatchedValue != null)
{
newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(currentYaraRulesMatchedValue));
}
newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(fileProperties.YaraMatchedRules));
_dataClient.UpdateExistingYaraRule(key, newYaraMatchedRules);
}
_dataClient.UpdatePrevalenceCount(key, count);
return(true);
}
