private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = node.GetBytes(); _timingMetrics.Start(TimingMetric.FileHashing); this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } _timingMetrics.Stop(TimingMetric.FileHashing); if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); _timingMetrics.Stop(TimingMetric.YaraScanning); compiledRules = null; } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { _timingMetrics.Start(TimingMetric.CalculatingEntropy); this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters) { YSScanner results = null; using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling)) { List <YaraFilter> yaraFilters = parameters.YaraParameters; List <string> distinctRulesToRun = yaraFilters .SelectMany(yf => yf.ProcessRule(this)) .Distinct() .ToList(); if (!distinctRulesToRun.Any()) { distinctRulesToRun = yaraFilters .Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch) .SelectMany(yf => yf.OnMatchRules) .Distinct() .ToList(); } if (!distinctRulesToRun.Any()) { return(null); } distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList(); string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun); string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken)); if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash)) { results = _yaraCompiledRulesDictionary[ruleCollectionHash]; } else { try { results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex); } _yaraCompiledRulesDictionary.Add(ruleCollectionHash, results); } } return(results); }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node) { if (parameters == null) { return; } if (node == null) { return; } CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); MFTNumber = node.MFTRecordNumber; SequenceNumber = node.SequenceNumber; DriveLetter = driveLetter; FileName = node.Name; MftTimeAccessed = node.LastAccessTime; MftTimeCreation = node.CreationTime; MftTimeModified = node.LastChangeTime; MftTimeMftModified = node.TimeMftModified; DirectoryLocation = Path.GetDirectoryName(node.FullName); Extension = Path.GetExtension(FileName); FullPath = node.FullName; this.Length = node.Size; CancellationHelper.ThrowIfCancelled(); if (this.Length == 0) //Workaround for hard links { if (System.IO.File.Exists(FullPath)) { long length = new System.IO.FileInfo(FullPath).Length; if (length > 0) { this.Length = (ulong)length; node.Size = this.Length; } } } bool haveFileReadPermission = true; var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath); if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read) { haveFileReadPermission = true; } if (this.Length != 0) { var maxLength = ((ulong)NtfsReader.MaxClustersToRead * (ulong)4096); // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length > maxLength) //(int.MaxValue - 1)) { IEnumerable <byte[]> fileChunks = node.GetBytes(); CancellationHelper.ThrowIfCancelled(); this.Sha256Hash = GetSha256Hash_IEnumerable(fileChunks, this.Length); CancellationHelper.ThrowIfCancelled(); if (haveFileReadPermission) { this.PeData = PeDataObject.TryGetPeDataObject(FullPath, parameters.OnlineCertValidation); CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FullPath); CancellationHelper.ThrowIfCancelled(); } /* * if (parameters.CalculateEntropy) * { * this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); * CancellationHelper.ThrowIfCancelled(); * } */ if (haveFileReadPermission) { if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.ScanFile(FullPath, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } } } else { byte[] fileBytes = new byte[0]; if (!node.Streams.Any()) //workaround for no file stream such as with hard links { try { using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read)) { // Read the source file into a byte array. fileBytes = new byte[fsSource.Length]; int numBytesToRead = (int)fsSource.Length; int numBytesRead = 0; while (numBytesToRead > 0) { // Read may return anything from 0 to numBytesToRead. int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead); // Break when the end of the file is reached. if (n == 0) { break; } numBytesRead += n; numBytesToRead -= n; } numBytesToRead = fileBytes.Length; } } catch (Exception ex) { } } else { fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray(); CancellationHelper.ThrowIfCancelled(); } this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation); if (IsPeDataPopulated) { this.Sha256Hash = PeData.SHA256Hash; } else { this.Sha256Hash = GetSha256Hash_Array(fileBytes); } CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes); CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); CancellationHelper.ThrowIfCancelled(); } if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.ScanBytes(fileBytes, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } } } this.Attributes = new Attributes(node); CancellationHelper.ThrowIfCancelled(); if (haveFileReadPermission) { PopulateFileInfoProperties(FullPath); CancellationHelper.ThrowIfCancelled(); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } }
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { _timingMetrics.Start(TimingMetric.ReadingMFTBytes); byte[] fileBytes = new byte[0]; if (!node.Streams.Any()) //workaround for no file stream such as with hard links { try { using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read)) { // Read the source file into a byte array. fileBytes = new byte[fsSource.Length]; int numBytesToRead = (int)fsSource.Length; int numBytesRead = 0; while (numBytesToRead > 0) { // Read may return anything from 0 to numBytesToRead. int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead); // Break when the end of the file is reached. if (n == 0) { break; } numBytesRead += n; numBytesToRead -= n; } numBytesToRead = fileBytes.Length; } } catch (System.IO.IOException ioException) { if (ioException.Message.Contains("contains a virus")) { string hash = "File access blocked by anti-virus program."; this.Sha256 = hash; this.SHA1 = hash; this.MD5 = hash; return; } } catch { } } else { fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray(); } _timingMetrics.Stop(TimingMetric.ReadingMFTBytes); _timingMetrics.Start(TimingMetric.FileHashing); this._peData = PeDataObject.TryGetPeDataObject(fileBytes); if (this._peData != null) { this.Sha256 = _peData.SHA256Hash; this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256)) { this.Sha256 = Hash.ByteArray.Sha256(fileBytes); this.SHA1 = Hash.ByteArray.Sha1(fileBytes); this.MD5 = Hash.ByteArray.MD5(fileBytes); } _timingMetrics.Stop(TimingMetric.FileHashing); _timingMetrics.Start(TimingMetric.MiscFileProperties); if (_peData != null) { _authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } _timingMetrics.Stop(TimingMetric.MiscFileProperties); if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); try { _yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex); } _timingMetrics.Stop(TimingMetric.YaraScanning); } CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { _timingMetrics.Start(TimingMetric.CalculatingEntropy); Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node, TimingMetrics timingMetrics) { if (parameters == null) { return; } if (node == null) { return; } _timingMetrics = timingMetrics; CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); _timingMetrics.Start(TimingMetric.MiscFileProperties); MFTNumber = node.MFTRecordNumber; SequenceNumber = node.SequenceNumber; DriveLetter = driveLetter; FileName = node.Name; MftTimeAccessed = node.LastAccessTime; MftTimeCreation = node.CreationTime; MftTimeModified = node.LastChangeTime; MftTimeMftModified = node.TimeMftModified; DirectoryLocation = Path.GetDirectoryName(node.FullName); Extension = Path.GetExtension(FileName); FullPath = node.FullName; this.Length = node.Size; CancellationHelper.ThrowIfCancelled(); if (this.Length == 0) //Workaround for hard links { if (File.Exists(FullPath)) { long length = new FileInfo(FullPath).Length; if (length > 0) { this.Length = (ulong)length; node.Size = this.Length; } } } bool hasFileReadPermissions = true; var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath); if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read) { hasFileReadPermissions = true; } _timingMetrics.Stop(TimingMetric.MiscFileProperties); if (this.Length != 0) { // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length > MaxLength) //(int.MaxValue - 1)) { PopulateLargeFile(parameters, node, hasFileReadPermissions); } else { PopulateSmallFile(parameters, node, hasFileReadPermissions); } } else { this.Sha256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"; // SHA256 hash of a null or zero-length input this.SHA1 = "DA39A3EE5E6B4B0D3255BFEF95601890AFD80709"; this.MD5 = "D41D8CD98F00B204E9800998ECF8427E"; } _timingMetrics.Start(TimingMetric.MiscFileProperties); PopulateIsTrusted(); CancellationHelper.ThrowIfCancelled(); this._attributes = new Attributes(node); _timingMetrics.Stop(TimingMetric.MiscFileProperties); CancellationHelper.ThrowIfCancelled(); }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, NtfsFile ntfsFile) { if (parameters == null) { return; } if (ntfsFile == null) { return; } CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); MFTNumber = ntfsFile.MFTRecord.MFTNumber; SequenceNumber = ntfsFile.MFTRecord.SequenceNumber; DriveLetter = driveLetter; FileName = ntfsFile.Name; MftTimeAccessed = ntfsFile.TimeAccessed; MftTimeCreation = ntfsFile.TimeCreation; MftTimeModified = ntfsFile.TimeModified; MftTimeMftModified = ntfsFile.TimeMftModified; DirectoryLocation = GetDirectory(ntfsFile); Extension = Path.GetExtension(FileName); FullPath = Path.Combine(DriveLetter.ToString() + DirectoryLocation, FileName); this.Length = ntfsFile.AllocatedSize; CancellationHelper.ThrowIfCancelled(); if (this.Length != 0) { // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length >= (int.MaxValue - 1)) { using (Stream ntfsStream = ntfsFile.OpenRead()) { this.Sha256Hash = GetSha256Hash_BigFile(ntfsStream); this.Length = ntfsStream.Length; } this.PeData = PeDataObject.TryGetPeDataObject(FileName, parameters.OnlineCertValidation); CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FileName); CancellationHelper.ThrowIfCancelled(); } else { byte[] fileBytes = GetFileBytes(ntfsFile); CancellationHelper.ThrowIfCancelled(); this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation); if (IsPeDataPopulated) { this.Sha256Hash = PeData.SHA256Hash; } else { this.Sha256Hash = GetSha256Hash(fileBytes); } CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); CancellationHelper.ThrowIfCancelled(); } this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes); CancellationHelper.ThrowIfCancelled(); if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.Scan(fileBytes, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } fileBytes = null; } } PopulateFileInfoProperties(FullPath); CancellationHelper.ThrowIfCancelled(); this.Attributes = new Attributes(FullPath); CancellationHelper.ThrowIfCancelled(); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); }
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = null; using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes)) { fileChunks = node.GetBytes(); } using (var timer = new TimingMetrics(TimingMetric.FileHashing)) { this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } } using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties)) { if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } } if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { using (var timer = new TimingMetrics(TimingMetric.YaraScanning)) { try { _yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex); } } } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy)) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); } CancellationHelper.ThrowIfCancelled(); } }