private static void YaraRulesCompileTest(FileEnumeratorParameters parameters) { if (!parameters.YaraParameters.Any()) { return; } foreach (YaraFilter filter in parameters.YaraParameters) { YSScanner compiledRule = null; try { compiledRule = YaraHelper.CompileRules(filter.OnMatchRules, parameters.ReportAndLogOutputFunction); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(YaraRulesCompileTest), string.Empty, ex); throw; } finally { if (compiledRule != null) { compiledRule.Dispose(); } } } }
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = node.GetBytes(); _timingMetrics.Start(TimingMetric.FileHashing); this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } _timingMetrics.Stop(TimingMetric.FileHashing); if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); _timingMetrics.Stop(TimingMetric.YaraScanning); compiledRules = null; } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { _timingMetrics.Start(TimingMetric.CalculatingEntropy); this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters) { YSScanner results = null; using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling)) { List <YaraFilter> yaraFilters = parameters.YaraParameters; List <string> distinctRulesToRun = yaraFilters .SelectMany(yf => yf.ProcessRule(this)) .Distinct() .ToList(); if (!distinctRulesToRun.Any()) { distinctRulesToRun = yaraFilters .Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch) .SelectMany(yf => yf.OnMatchRules) .Distinct() .ToList(); } if (!distinctRulesToRun.Any()) { return(null); } distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList(); string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun); string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken)); if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash)) { results = _yaraCompiledRulesDictionary[ruleCollectionHash]; } else { try { results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex); } _yaraCompiledRulesDictionary.Add(ruleCollectionHash, results); } } return(results); }
public void UpdateExistingYaraRule(SqlKey key, List <string> newYaraMatchedRules) { string newYaraRulesMatchedValue = YaraHelper.FormatDelimitedRulesString(newYaraMatchedRules); SQLiteParameter yaraMatchedRulesParameter = SqlHelper.GetParameter("YaraRulesMatched", newYaraRulesMatchedValue); List <SQLiteParameter> parameters = key.GetSqlParameters().ToList(); parameters.Add(yaraMatchedRulesParameter); string commandText = string.Format(SqlStrings.UpdateYaraRules, yaraMatchedRulesParameter.ParameterName); ExecuteNonQuery(commandText, parameters); }
private bool InsertIntoDB(FileProperties fileProperties, SqlKey key, List <SqlParameter> sqlParameters) { string updateText = string.Empty; if (!string.IsNullOrWhiteSpace(fileProperties.YaraMatchedRules)) { List <string> newYaraMatchedRules = new List <string>(); string currentYaraRulesMatchedValue = GetExistingYaraRules(key); if (currentYaraRulesMatchedValue != null) { newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(currentYaraRulesMatchedValue)); } newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(fileProperties.YaraMatchedRules)); string newYaraRulesMatchedValue = YaraHelper.FormatDelimitedRulesString(newYaraMatchedRules); SqlParameter yaraMatchedValueParameter = ParameterHelper.GetNewStringParameter("YaraRulesMatched", newYaraRulesMatchedValue); sqlParameters.Add(yaraMatchedValueParameter); updateText = $"UPDATE [{TableName}] SET [YaraRulesMatched] = {yaraMatchedValueParameter.ParameterName} WHERE {key.GetWhereClause()}"; } string columnNames = sqlParameters.AsColumnString(); string values = sqlParameters.AsValuesString(); string insertStatement = $@" INSERT INTO [{TableName}] ({columnNames},[PrevalenceCount],[DateSeen]) VALUES ({values},1,GETDATE())" ; string commandText = $@"DECLARE @PREVALENCECOUNT INT; SET @PREVALENCECOUNT = ( SELECT [PrevalenceCount] FROM [{TableName}] WHERE {key.GetWhereClause()} ) SET @PREVALENCECOUNT = @PREVALENCECOUNT + 1; IF(@PREVALENCECOUNT IS NOT NULL) BEGIN UPDATE [{TableName}] SET [PrevalenceCount] = @PREVALENCECOUNT WHERE {key.GetWhereClause()} {updateText} END ELSE BEGIN {insertStatement} END "; return(ExecNonQuery(commandText, sqlParameters)); }
public static void CleanUp() { if (_yaraCompiledRulesDictionary.Any()) { List <YSScanner> rules = _yaraCompiledRulesDictionary.Values.ToList(); foreach (YSScanner scanner in rules) { scanner.Dispose(); } _yaraCompiledRulesDictionary.Clear(); _yaraCompiledRulesDictionary = null; } YaraHelper.CleanUp(); }
private string PopulateYaraInfo(List <YaraFilter> yaraFilters) { List <string> distinctRulesToRun = yaraFilters .SelectMany(yf => yf.ProcessRule(this)) .Distinct() .OrderBy(s => s) .ToList(); if (!distinctRulesToRun.Any()) { distinctRulesToRun = yaraFilters .Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch) .SelectMany(yf => yf.OnMatchRules) .Distinct() .ToList(); } if (!distinctRulesToRun.Any()) { return(string.Empty); } _timingMetrics.Start(TimingMetric.YaraIndexBuilding); string yaraIndexContents = YaraHelper.MakeYaraIndexFile(distinctRulesToRun); string indexFileHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(yaraIndexContents)); string yaraIndexFilename = Path.Combine(Path.GetTempPath(), $"{indexFileHash}-index.yar"); if (!File.Exists(yaraIndexFilename)) { File.WriteAllText(yaraIndexFilename, yaraIndexContents); } _timingMetrics.Stop(TimingMetric.YaraIndexBuilding); return(yaraIndexFilename); }
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { _timingMetrics.Start(TimingMetric.ReadingMFTBytes); byte[] fileBytes = new byte[0]; if (!node.Streams.Any()) //workaround for no file stream such as with hard links { try { using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read)) { // Read the source file into a byte array. fileBytes = new byte[fsSource.Length]; int numBytesToRead = (int)fsSource.Length; int numBytesRead = 0; while (numBytesToRead > 0) { // Read may return anything from 0 to numBytesToRead. int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead); // Break when the end of the file is reached. if (n == 0) { break; } numBytesRead += n; numBytesToRead -= n; } numBytesToRead = fileBytes.Length; } } catch (System.IO.IOException ioException) { if (ioException.Message.Contains("contains a virus")) { string hash = "File access blocked by anti-virus program."; this.Sha256 = hash; this.SHA1 = hash; this.MD5 = hash; return; } } catch { } } else { fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray(); } _timingMetrics.Stop(TimingMetric.ReadingMFTBytes); _timingMetrics.Start(TimingMetric.FileHashing); this._peData = PeDataObject.TryGetPeDataObject(fileBytes); if (this._peData != null) { this.Sha256 = _peData.SHA256Hash; this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256)) { this.Sha256 = Hash.ByteArray.Sha256(fileBytes); this.SHA1 = Hash.ByteArray.Sha1(fileBytes); this.MD5 = Hash.ByteArray.MD5(fileBytes); } _timingMetrics.Stop(TimingMetric.FileHashing); _timingMetrics.Start(TimingMetric.MiscFileProperties); if (_peData != null) { _authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } _timingMetrics.Stop(TimingMetric.MiscFileProperties); if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); try { _yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex); } _timingMetrics.Stop(TimingMetric.YaraScanning); } CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { _timingMetrics.Start(TimingMetric.CalculatingEntropy); Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = null; using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes)) { fileChunks = node.GetBytes(); } using (var timer = new TimingMetrics(TimingMetric.FileHashing)) { this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } } using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties)) { if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } } if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { using (var timer = new TimingMetrics(TimingMetric.YaraScanning)) { try { _yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex); } } } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy)) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); } CancellationHelper.ThrowIfCancelled(); } }
public bool PersistFileProperties(FileProperties fileProperties) { SqlKey key = new SqlKey(fileProperties.MFTNumber, fileProperties.SequenceNumber, fileProperties.Sha256); List <SQLiteParameter> sqlParameters = new List <SQLiteParameter>(); sqlParameters.AddRange(key.GetSqlParameters()); sqlParameters.AddRange(new List <SQLiteParameter> { SqlHelper.GetParameter("DriveLetter", fileProperties.DriveLetter), SqlHelper.GetParameter("FullPath", fileProperties.FullPath), SqlHelper.GetParameter("Filename", fileProperties.FileName), SqlHelper.GetParameter("Extension", fileProperties.Extension), SqlHelper.GetParameter("DirectoryLocation", fileProperties.DirectoryLocation), SqlHelper.GetParameter("Length", fileProperties.Length), SqlHelper.GetParameter("MftTimeCreation", fileProperties.MftTimeCreation), SqlHelper.GetParameter("MftTimeAccessed", fileProperties.MftTimeAccessed), SqlHelper.GetParameter("MftTimeModified", fileProperties.MftTimeModified), SqlHelper.GetParameter("MftTimeMftModified", fileProperties.MftTimeMftModified), SqlHelper.GetParameter("CreationTime", fileProperties.CreationTime), SqlHelper.GetParameter("LastAccessTime", fileProperties.LastAccessTime), SqlHelper.GetParameter("LastWriteTime", fileProperties.LastWriteTime), SqlHelper.GetParameter("Project", fileProperties.Project), SqlHelper.GetParameter("ProviderItemID", fileProperties.ProviderItemID), SqlHelper.GetParameter("OriginalFileName", fileProperties.OriginalFileName), SqlHelper.GetParameter("FileOwner", fileProperties.FileOwner), SqlHelper.GetParameter("FileVersion", fileProperties.FileVersion), SqlHelper.GetParameter("FileDescription", fileProperties.FileDescription), SqlHelper.GetParameter("Trademarks", fileProperties.Trademarks), SqlHelper.GetParameter("Copyright", fileProperties.Copyright), SqlHelper.GetParameter("Company", fileProperties.Company), SqlHelper.GetParameter("ApplicationName", fileProperties.ApplicationName), SqlHelper.GetParameter("Comment", fileProperties.Comment), SqlHelper.GetParameter("Title", fileProperties.Title), SqlHelper.GetParameter("Link", fileProperties.Link), SqlHelper.GetParameter("MimeType", fileProperties.MimeType), SqlHelper.GetParameter("InternalName", fileProperties.InternalName), SqlHelper.GetParameter("ProductName", fileProperties.ProductName), SqlHelper.GetParameter("Language", fileProperties.Language), SqlHelper.GetParameter("ComputerName", fileProperties.ComputerName), SqlHelper.GetParameter("Attributes", fileProperties.Attributes), SqlHelper.GetParameter("SHA1", fileProperties.SHA1), SqlHelper.GetParameter("MD5", fileProperties.MD5), SqlHelper.GetParameter("ImpHash", fileProperties.ImpHash), SqlHelper.GetParameter("IsDll", fileProperties.IsDll), SqlHelper.GetParameter("IsExe", fileProperties.IsExe), SqlHelper.GetParameter("IsDriver", fileProperties.IsDriver), SqlHelper.GetParameter("IsSigned", fileProperties.IsSigned), SqlHelper.GetParameter("IsSignatureValid", fileProperties.IsSignatureValid), SqlHelper.GetParameter("IsValidCertChain", fileProperties.IsValidCertChain), SqlHelper.GetNewParameterByType("BinaryType", fileProperties.BinaryType.GetValueOrDefault(), DbType.Int32), SqlHelper.GetNewParameterByType("CompileDate", fileProperties.CompileDate.GetValueOrDefault(), DbType.DateTime2), SqlHelper.GetParameter("IsTrusted", fileProperties.IsTrusted), SqlHelper.GetParameter("CertSubject", fileProperties.CertSubject), SqlHelper.GetParameter("CertIssuer", fileProperties.CertIssuer), SqlHelper.GetParameter("CertSerialNumber", fileProperties.CertSerialNumber), SqlHelper.GetParameter("CertThumbprint", fileProperties.CertThumbprint), SqlHelper.GetParameter("CertNotBefore", fileProperties.CertNotBefore), SqlHelper.GetParameter("CertNotAfter", fileProperties.CertNotAfter), SqlHelper.GetParameter("Entropy", fileProperties.Entropy ?? 0) }); int count = _dataClient.GetPrevalenceCount(key); if (count == -1) { _dataClient.InsertRow(sqlParameters); return(true); } count += 1; if (!string.IsNullOrWhiteSpace(fileProperties.YaraMatchedRules)) { List <string> newYaraMatchedRules = new List <string>(); string currentYaraRulesMatchedValue = _dataClient.GetExistingYaraRules(key); if (currentYaraRulesMatchedValue != null) { newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(currentYaraRulesMatchedValue)); } newYaraMatchedRules.AddRange(YaraHelper.ParseDelimitedRulesString(fileProperties.YaraMatchedRules)); _dataClient.UpdateExistingYaraRule(key, newYaraMatchedRules); } _dataClient.UpdatePrevalenceCount(key, count); return(true); }