private static ulong GetCompressedLength(IEnumerable <byte[]> fileChunks) { ulong result = 0; try { using (MemoryStream outStream = new MemoryStream()) { using (GZipStream gzip = new GZipStream(outStream, CompressionMode.Compress)) { using (MemoryStream inStream = new MemoryStream(Int16.MaxValue)) { foreach (byte[] chunk in fileChunks) { inStream.Write(chunk, 0, chunk.Length); } Task copyTask = inStream.CopyToAsync(gzip, chunkSize, CancellationHelper.GetCancellationToken()); copyTask.Wait(CancellationHelper.GetCancellationToken()); result = (ulong)outStream.Length; } } } } catch { } return(result); }
private string GetSha256Hash_BigFile(Stream fileStream) { string result = string.Empty; using (SHA256Managed hashAlgorithm = new SHA256Managed()) { long bytesToHash = fileStream.Length; byte[] buffer = new byte[bigChunkSize]; int sizeToRead = buffer.Length; while (bytesToHash > 0) { if (bytesToHash < (long)sizeToRead) { sizeToRead = (int)bytesToHash; } int bytesRead = fileStream.ReadAsync(buffer, 0, sizeToRead, CancellationHelper.GetCancellationToken()).Result; CancellationHelper.ThrowIfCancelled(); hashAlgorithm.TransformBlock(buffer, 0, bytesRead, null, 0); bytesToHash -= (long)bytesRead; if (bytesRead == 0) { throw new InvalidOperationException("Unexpected end of stream"); // or break; } } hashAlgorithm.TransformFinalBlock(buffer, 0, 0); buffer = null; result = ByteArrayConverter.ToHexString(hashAlgorithm.Hash); } return(result); }
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = node.GetBytes(); _timingMetrics.Start(TimingMetric.FileHashing); this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } _timingMetrics.Stop(TimingMetric.FileHashing); if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); _timingMetrics.Stop(TimingMetric.YaraScanning); compiledRules = null; } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { _timingMetrics.Start(TimingMetric.CalculatingEntropy); this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
private static string FormatResults(List <ScanResult> scanResults) { string result = null; CancellationHelper.ThrowIfCancelled(); if (scanResults.Any()) { IEnumerable <string> matchingRules = scanResults.Select(res => res.MatchingRule.Identifier); CancellationHelper.ThrowIfCancelled(); if (matchingRules.Any()) { result = string.Join("|", matchingRules); } } return(result); }
public static double?CalculateFileEntropy(byte[] fileBytes) { double?result = null; CancellationHelper.ThrowIfCancelled(); ulong uncompressedLength = (ulong)fileBytes.LongLength; if (uncompressedLength > 20) { ulong compressedLength = GetCompressedLength(fileBytes); if (compressedLength > 0) { CancellationHelper.ThrowIfCancelled(); result = CalculateRatio(uncompressedLength, compressedLength); } } return(result); }
public static double?CalculateFileEntropy(IEnumerable <byte[]> fileChunks, ulong fileSize) { double?result = null; CancellationHelper.ThrowIfCancelled(); ulong uncompressedLength = fileSize; if (uncompressedLength > 20) { ulong compressedLength = GetCompressedLength(fileChunks); if (compressedLength > 0) { CancellationHelper.ThrowIfCancelled(); result = CalculateRatio(uncompressedLength, compressedLength); } } return(result); }
private void PopulateShellFileInfo(string fullPath) { try { if (!ShellFile.IsPlatformSupported) { return; } using (var timer = new TimingMetrics(TimingMetric.GettingShellInfo)) { using (ShellFile file = ShellFile.FromFilePath(fullPath)) { using (ShellProperties fileProperties = file.Properties) { ShellProperties.PropertySystem shellProperty = fileProperties.System; Project = shellProperty.Project.Value ?? ""; ProviderItemID = shellProperty.ProviderItemID.Value ?? ""; OriginalFileName = shellProperty.OriginalFileName.Value ?? ""; FileOwner = shellProperty.FileOwner.Value ?? ""; FileVersion = shellProperty.FileVersion.Value ?? ""; FileDescription = shellProperty.FileDescription.Value ?? ""; Trademarks = shellProperty.Trademarks.Value ?? ""; Link = shellProperty.Link.TargetUrl.Value ?? ""; Copyright = shellProperty.Copyright.Value ?? ""; Company = shellProperty.Company.Value ?? ""; ApplicationName = shellProperty.ApplicationName.Value ?? ""; Comment = shellProperty.Comment.Value ?? ""; Title = shellProperty.Title.Value ?? ""; CancellationHelper.ThrowIfCancelled(); MimeType = shellProperty.ContentType.Value ?? ""; InternalName = shellProperty.InternalName.Value ?? ""; ProductName = shellProperty.Software.ProductName.Value ?? ""; Language = shellProperty.Language.Value ?? ""; ComputerName = shellProperty.ComputerName.Value ?? ""; } } } } catch { } }
private byte[] GetFileBytes(NtfsFile file) { byte[] result = new byte[] { }; // If we made it this far, attempt to read the bytes using (MemoryStream ms = new MemoryStream((int)this.Length)) { using (Stream ntfsStream = file.OpenRead()) { int read; byte[] buffer = new byte[16 * 1024]; while ((read = ntfsStream.Read(buffer, 0, buffer.Length)) > 0) { ms.Write(buffer, 0, read); CancellationHelper.ThrowIfCancelled(); } buffer = null; } result = ms.ToArray(); } return(result); }
private static long GetCompressedLength(byte[] fileBytes) { long result = 0; try { using (MemoryStream outStream = new MemoryStream()) { using (GZipStream gzip = new GZipStream(outStream, CompressionMode.Compress)) { using (MemoryStream byteStream = new MemoryStream(fileBytes)) { byteStream.CopyToAsync(gzip, 81920, CancellationHelper.GetCancellationToken()).Wait(CancellationHelper.GetCancellationToken()); } result = outStream.Length; } } } catch { } return(result); }
public static string Scan(byte[] fileBytes, string rulesPath) { string result = null; try { List <ScanResult> scanResults = QuickScan.Memory(fileBytes, rulesPath); CancellationHelper.ThrowIfCancelled(); if (scanResults.Any()) { IEnumerable <string> matchingRules = scanResults.Select(res => res.MatchingRule.Identifier); CancellationHelper.ThrowIfCancelled(); if (matchingRules.Any()) { result = string.Join("|", matchingRules); } } } catch { } return(result); }
private static ulong GetCompressedLength(byte[] fileBytes) { ulong result = 0; try { using (MemoryStream outStream = new MemoryStream()) { using (GZipStream gzip = new GZipStream(outStream, CompressionMode.Compress)) { using (MemoryStream inStream = new MemoryStream(fileBytes)) { Task copyTask = inStream.CopyToAsync(gzip, chunkSize, CancellationHelper.GetCancellationToken()); copyTask.Wait(CancellationHelper.GetCancellationToken()); result = (ulong)outStream.Length; } } } } catch { } return(result); }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node) { if (parameters == null) { return; } if (node == null) { return; } CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); MFTNumber = node.MFTRecordNumber; SequenceNumber = node.SequenceNumber; DriveLetter = driveLetter; FileName = node.Name; MftTimeAccessed = node.LastAccessTime; MftTimeCreation = node.CreationTime; MftTimeModified = node.LastChangeTime; MftTimeMftModified = node.TimeMftModified; DirectoryLocation = Path.GetDirectoryName(node.FullName); Extension = Path.GetExtension(FileName); FullPath = node.FullName; this.Length = node.Size; CancellationHelper.ThrowIfCancelled(); if (this.Length == 0) //Workaround for hard links { if (System.IO.File.Exists(FullPath)) { long length = new System.IO.FileInfo(FullPath).Length; if (length > 0) { this.Length = (ulong)length; node.Size = this.Length; } } } bool haveFileReadPermission = true; var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath); if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read) { haveFileReadPermission = true; } if (this.Length != 0) { var maxLength = ((ulong)NtfsReader.MaxClustersToRead * (ulong)4096); // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length > maxLength) //(int.MaxValue - 1)) { IEnumerable <byte[]> fileChunks = node.GetBytes(); CancellationHelper.ThrowIfCancelled(); this.Sha256Hash = GetSha256Hash_IEnumerable(fileChunks, this.Length); CancellationHelper.ThrowIfCancelled(); if (haveFileReadPermission) { this.PeData = PeDataObject.TryGetPeDataObject(FullPath, parameters.OnlineCertValidation); CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FullPath); CancellationHelper.ThrowIfCancelled(); } /* * if (parameters.CalculateEntropy) * { * this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); * CancellationHelper.ThrowIfCancelled(); * } */ if (haveFileReadPermission) { if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.ScanFile(FullPath, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } } } else { byte[] fileBytes = new byte[0]; if (!node.Streams.Any()) //workaround for no file stream such as with hard links { try { using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read)) { // Read the source file into a byte array. fileBytes = new byte[fsSource.Length]; int numBytesToRead = (int)fsSource.Length; int numBytesRead = 0; while (numBytesToRead > 0) { // Read may return anything from 0 to numBytesToRead. int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead); // Break when the end of the file is reached. if (n == 0) { break; } numBytesRead += n; numBytesToRead -= n; } numBytesToRead = fileBytes.Length; } } catch (Exception ex) { } } else { fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray(); CancellationHelper.ThrowIfCancelled(); } this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation); if (IsPeDataPopulated) { this.Sha256Hash = PeData.SHA256Hash; } else { this.Sha256Hash = GetSha256Hash_Array(fileBytes); } CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes); CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); CancellationHelper.ThrowIfCancelled(); } if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.ScanBytes(fileBytes, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } } } this.Attributes = new Attributes(node); CancellationHelper.ThrowIfCancelled(); if (haveFileReadPermission) { PopulateFileInfoProperties(FullPath); CancellationHelper.ThrowIfCancelled(); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } }
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { _timingMetrics.Start(TimingMetric.ReadingMFTBytes); byte[] fileBytes = new byte[0]; if (!node.Streams.Any()) //workaround for no file stream such as with hard links { try { using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read)) { // Read the source file into a byte array. fileBytes = new byte[fsSource.Length]; int numBytesToRead = (int)fsSource.Length; int numBytesRead = 0; while (numBytesToRead > 0) { // Read may return anything from 0 to numBytesToRead. int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead); // Break when the end of the file is reached. if (n == 0) { break; } numBytesRead += n; numBytesToRead -= n; } numBytesToRead = fileBytes.Length; } } catch (System.IO.IOException ioException) { if (ioException.Message.Contains("contains a virus")) { string hash = "File access blocked by anti-virus program."; this.Sha256 = hash; this.SHA1 = hash; this.MD5 = hash; return; } } catch { } } else { fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray(); } _timingMetrics.Stop(TimingMetric.ReadingMFTBytes); _timingMetrics.Start(TimingMetric.FileHashing); this._peData = PeDataObject.TryGetPeDataObject(fileBytes); if (this._peData != null) { this.Sha256 = _peData.SHA256Hash; this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256)) { this.Sha256 = Hash.ByteArray.Sha256(fileBytes); this.SHA1 = Hash.ByteArray.Sha1(fileBytes); this.MD5 = Hash.ByteArray.MD5(fileBytes); } _timingMetrics.Stop(TimingMetric.FileHashing); _timingMetrics.Start(TimingMetric.MiscFileProperties); if (_peData != null) { _authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } _timingMetrics.Stop(TimingMetric.MiscFileProperties); if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { _timingMetrics.Start(TimingMetric.YaraScanning); try { _yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex); } _timingMetrics.Stop(TimingMetric.YaraScanning); } CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { _timingMetrics.Start(TimingMetric.CalculatingEntropy); Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); _timingMetrics.Stop(TimingMetric.CalculatingEntropy); CancellationHelper.ThrowIfCancelled(); } }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node, TimingMetrics timingMetrics) { if (parameters == null) { return; } if (node == null) { return; } _timingMetrics = timingMetrics; CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); _timingMetrics.Start(TimingMetric.MiscFileProperties); MFTNumber = node.MFTRecordNumber; SequenceNumber = node.SequenceNumber; DriveLetter = driveLetter; FileName = node.Name; MftTimeAccessed = node.LastAccessTime; MftTimeCreation = node.CreationTime; MftTimeModified = node.LastChangeTime; MftTimeMftModified = node.TimeMftModified; DirectoryLocation = Path.GetDirectoryName(node.FullName); Extension = Path.GetExtension(FileName); FullPath = node.FullName; this.Length = node.Size; CancellationHelper.ThrowIfCancelled(); if (this.Length == 0) //Workaround for hard links { if (File.Exists(FullPath)) { long length = new FileInfo(FullPath).Length; if (length > 0) { this.Length = (ulong)length; node.Size = this.Length; } } } bool hasFileReadPermissions = true; var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath); if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read) { hasFileReadPermissions = true; } _timingMetrics.Stop(TimingMetric.MiscFileProperties); if (this.Length != 0) { // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length > MaxLength) //(int.MaxValue - 1)) { PopulateLargeFile(parameters, node, hasFileReadPermissions); } else { PopulateSmallFile(parameters, node, hasFileReadPermissions); } } else { this.Sha256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"; // SHA256 hash of a null or zero-length input this.SHA1 = "DA39A3EE5E6B4B0D3255BFEF95601890AFD80709"; this.MD5 = "D41D8CD98F00B204E9800998ECF8427E"; } _timingMetrics.Start(TimingMetric.MiscFileProperties); PopulateIsTrusted(); CancellationHelper.ThrowIfCancelled(); this._attributes = new Attributes(node); _timingMetrics.Stop(TimingMetric.MiscFileProperties); CancellationHelper.ThrowIfCancelled(); }
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, NtfsFile ntfsFile) { if (parameters == null) { return; } if (ntfsFile == null) { return; } CancellationHelper.SetCancellationToken(parameters.CancelToken); CancellationHelper.ThrowIfCancelled(); MFTNumber = ntfsFile.MFTRecord.MFTNumber; SequenceNumber = ntfsFile.MFTRecord.SequenceNumber; DriveLetter = driveLetter; FileName = ntfsFile.Name; MftTimeAccessed = ntfsFile.TimeAccessed; MftTimeCreation = ntfsFile.TimeCreation; MftTimeModified = ntfsFile.TimeModified; MftTimeMftModified = ntfsFile.TimeMftModified; DirectoryLocation = GetDirectory(ntfsFile); Extension = Path.GetExtension(FileName); FullPath = Path.Combine(DriveLetter.ToString() + DirectoryLocation, FileName); this.Length = ntfsFile.AllocatedSize; CancellationHelper.ThrowIfCancelled(); if (this.Length != 0) { // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here. if (this.Length >= (int.MaxValue - 1)) { using (Stream ntfsStream = ntfsFile.OpenRead()) { this.Sha256Hash = GetSha256Hash_BigFile(ntfsStream); this.Length = ntfsStream.Length; } this.PeData = PeDataObject.TryGetPeDataObject(FileName, parameters.OnlineCertValidation); CancellationHelper.ThrowIfCancelled(); this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FileName); CancellationHelper.ThrowIfCancelled(); } else { byte[] fileBytes = GetFileBytes(ntfsFile); CancellationHelper.ThrowIfCancelled(); this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation); if (IsPeDataPopulated) { this.Sha256Hash = PeData.SHA256Hash; } else { this.Sha256Hash = GetSha256Hash(fileBytes); } CancellationHelper.ThrowIfCancelled(); if (parameters.CalculateEntropy) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes); CancellationHelper.ThrowIfCancelled(); } this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes); CancellationHelper.ThrowIfCancelled(); if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath)) { this.YaraRulesMatched = YaraRules.Scan(fileBytes, parameters.YaraRulesFilePath); CancellationHelper.ThrowIfCancelled(); } fileBytes = null; } } PopulateFileInfoProperties(FullPath); CancellationHelper.ThrowIfCancelled(); this.Attributes = new Attributes(FullPath); CancellationHelper.ThrowIfCancelled(); PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); }
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions) { IEnumerable <byte[]> fileChunks = null; using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes)) { fileChunks = node.GetBytes(); } using (var timer = new TimingMetrics(TimingMetric.FileHashing)) { this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks); if (hasFileReadPermissions) { this._peData = PeDataObject.TryGetPeDataObject(FullPath); if (_peData != null) { this.SHA1 = _peData.SHA1Hash; this.MD5 = _peData.MD5Hash; } else { this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks); this.MD5 = Hash.ByteEnumerable.MD5(fileChunks); } } } using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties)) { if (_peData != null) { this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate); } CancellationHelper.ThrowIfCancelled(); if (hasFileReadPermissions) { PopulateFileInfoProperties(FullPath); } } if (hasFileReadPermissions) { PopulateShellFileInfo(FullPath); CancellationHelper.ThrowIfCancelled(); } if (hasFileReadPermissions) { YSScanner compiledRules = GetCompiledYaraRules(parameters); if (compiledRules != null) { using (var timer = new TimingMetrics(TimingMetric.YaraScanning)) { try { _yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules); } catch (Exception ex) { parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex); } } } CancellationHelper.ThrowIfCancelled(); } if (parameters.CalculateEntropy) // Should we calculate entropy on really large files? { using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy)) { this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length); } CancellationHelper.ThrowIfCancelled(); } }