private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
IEnumerable <byte[]> fileChunks = node.GetBytes();
_timingMetrics.Start(TimingMetric.FileHashing);
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
_timingMetrics.Stop(TimingMetric.FileHashing);
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
_timingMetrics.Stop(TimingMetric.YaraScanning);
compiledRules = null;
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters)
{
YSScanner results = null;
using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling))
{
List <YaraFilter> yaraFilters = parameters.YaraParameters;
List <string> distinctRulesToRun =
yaraFilters
.SelectMany(yf => yf.ProcessRule(this))
.Distinct()
.ToList();
if (!distinctRulesToRun.Any())
{
distinctRulesToRun =
yaraFilters
.Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
.SelectMany(yf => yf.OnMatchRules)
.Distinct()
.ToList();
}
if (!distinctRulesToRun.Any())
{
return(null);
}
distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList();
string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun);
string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken));
if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash))
{
results = _yaraCompiledRulesDictionary[ruleCollectionHash];
}
else
{
try
{
results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex);
}
_yaraCompiledRulesDictionary.Add(ruleCollectionHash, results);
}
}
return(results);
}
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node)
{
if (parameters == null)
{
return;
}
if (node == null)
{
return;
}
CancellationHelper.SetCancellationToken(parameters.CancelToken);
CancellationHelper.ThrowIfCancelled();
MFTNumber = node.MFTRecordNumber;
SequenceNumber = node.SequenceNumber;
DriveLetter = driveLetter;
FileName = node.Name;
MftTimeAccessed = node.LastAccessTime;
MftTimeCreation = node.CreationTime;
MftTimeModified = node.LastChangeTime;
MftTimeMftModified = node.TimeMftModified;
DirectoryLocation = Path.GetDirectoryName(node.FullName);
Extension = Path.GetExtension(FileName);
FullPath = node.FullName;
this.Length = node.Size;
CancellationHelper.ThrowIfCancelled();
if (this.Length == 0) //Workaround for hard links
{
if (System.IO.File.Exists(FullPath))
{
long length = new System.IO.FileInfo(FullPath).Length;
if (length > 0)
{
this.Length = (ulong)length;
node.Size = this.Length;
}
}
}
bool haveFileReadPermission = true;
var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath);
if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read)
{
haveFileReadPermission = true;
}
if (this.Length != 0)
{
var maxLength = ((ulong)NtfsReader.MaxClustersToRead * (ulong)4096);
// Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
if (this.Length > maxLength) //(int.MaxValue - 1))
{
IEnumerable <byte[]> fileChunks = node.GetBytes();
CancellationHelper.ThrowIfCancelled();
this.Sha256Hash = GetSha256Hash_IEnumerable(fileChunks, this.Length);
CancellationHelper.ThrowIfCancelled();
if (haveFileReadPermission)
{
this.PeData = PeDataObject.TryGetPeDataObject(FullPath, parameters.OnlineCertValidation);
CancellationHelper.ThrowIfCancelled();
this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FullPath);
CancellationHelper.ThrowIfCancelled();
}
/*
* if (parameters.CalculateEntropy)
* {
* this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
* CancellationHelper.ThrowIfCancelled();
* }
*/
if (haveFileReadPermission)
{
if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
{
this.YaraRulesMatched = YaraRules.ScanFile(FullPath, parameters.YaraRulesFilePath);
CancellationHelper.ThrowIfCancelled();
}
}
}
else
{
byte[] fileBytes = new byte[0];
if (!node.Streams.Any()) //workaround for no file stream such as with hard links
{
try
{
using (FileStream fsSource = new FileStream(FullPath,
FileMode.Open, FileAccess.Read))
{
// Read the source file into a byte array.
fileBytes = new byte[fsSource.Length];
int numBytesToRead = (int)fsSource.Length;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
{
break;
}
numBytesRead += n;
numBytesToRead -= n;
}
numBytesToRead = fileBytes.Length;
}
}
catch (Exception ex)
{
}
}
else
{
fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
CancellationHelper.ThrowIfCancelled();
}
this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation);
if (IsPeDataPopulated)
{
this.Sha256Hash = PeData.SHA256Hash;
}
else
{
this.Sha256Hash = GetSha256Hash_Array(fileBytes);
}
CancellationHelper.ThrowIfCancelled();
this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes);
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
CancellationHelper.ThrowIfCancelled();
}
if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
{
this.YaraRulesMatched = YaraRules.ScanBytes(fileBytes, parameters.YaraRulesFilePath);
CancellationHelper.ThrowIfCancelled();
}
}
}
this.Attributes = new Attributes(node);
CancellationHelper.ThrowIfCancelled();
if (haveFileReadPermission)
{
PopulateFileInfoProperties(FullPath);
CancellationHelper.ThrowIfCancelled();
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
}
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
_timingMetrics.Start(TimingMetric.ReadingMFTBytes);
byte[] fileBytes = new byte[0];
if (!node.Streams.Any()) //workaround for no file stream such as with hard links
{
try
{
using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read))
{
// Read the source file into a byte array.
fileBytes = new byte[fsSource.Length];
int numBytesToRead = (int)fsSource.Length;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
{
break;
}
numBytesRead += n;
numBytesToRead -= n;
}
numBytesToRead = fileBytes.Length;
}
}
catch (System.IO.IOException ioException)
{
if (ioException.Message.Contains("contains a virus"))
{
string hash = "File access blocked by anti-virus program.";
this.Sha256 = hash;
this.SHA1 = hash;
this.MD5 = hash;
return;
}
}
catch
{ }
}
else
{
fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
}
_timingMetrics.Stop(TimingMetric.ReadingMFTBytes);
_timingMetrics.Start(TimingMetric.FileHashing);
this._peData = PeDataObject.TryGetPeDataObject(fileBytes);
if (this._peData != null)
{
this.Sha256 = _peData.SHA256Hash;
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256))
{
this.Sha256 = Hash.ByteArray.Sha256(fileBytes);
this.SHA1 = Hash.ByteArray.Sha1(fileBytes);
this.MD5 = Hash.ByteArray.MD5(fileBytes);
}
_timingMetrics.Stop(TimingMetric.FileHashing);
_timingMetrics.Start(TimingMetric.MiscFileProperties);
if (_peData != null)
{
_authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
_timingMetrics.Start(TimingMetric.YaraScanning);
try
{
_yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex);
}
_timingMetrics.Stop(TimingMetric.YaraScanning);
}
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
_timingMetrics.Start(TimingMetric.CalculatingEntropy);
Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
_timingMetrics.Stop(TimingMetric.CalculatingEntropy);
CancellationHelper.ThrowIfCancelled();
}
}
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node, TimingMetrics timingMetrics)
{
if (parameters == null)
{
return;
}
if (node == null)
{
return;
}
_timingMetrics = timingMetrics;
CancellationHelper.SetCancellationToken(parameters.CancelToken);
CancellationHelper.ThrowIfCancelled();
_timingMetrics.Start(TimingMetric.MiscFileProperties);
MFTNumber = node.MFTRecordNumber;
SequenceNumber = node.SequenceNumber;
DriveLetter = driveLetter;
FileName = node.Name;
MftTimeAccessed = node.LastAccessTime;
MftTimeCreation = node.CreationTime;
MftTimeModified = node.LastChangeTime;
MftTimeMftModified = node.TimeMftModified;
DirectoryLocation = Path.GetDirectoryName(node.FullName);
Extension = Path.GetExtension(FileName);
FullPath = node.FullName;
this.Length = node.Size;
CancellationHelper.ThrowIfCancelled();
if (this.Length == 0) //Workaround for hard links
{
if (File.Exists(FullPath))
{
long length = new FileInfo(FullPath).Length;
if (length > 0)
{
this.Length = (ulong)length;
node.Size = this.Length;
}
}
}
bool hasFileReadPermissions = true;
var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath);
if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read)
{
hasFileReadPermissions = true;
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (this.Length != 0)
{
// Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
if (this.Length > MaxLength) //(int.MaxValue - 1))
{
PopulateLargeFile(parameters, node, hasFileReadPermissions);
}
else
{
PopulateSmallFile(parameters, node, hasFileReadPermissions);
}
}
else
{
this.Sha256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"; // SHA256 hash of a null or zero-length input
this.SHA1 = "DA39A3EE5E6B4B0D3255BFEF95601890AFD80709";
this.MD5 = "D41D8CD98F00B204E9800998ECF8427E";
}
_timingMetrics.Start(TimingMetric.MiscFileProperties);
PopulateIsTrusted();
CancellationHelper.ThrowIfCancelled();
this._attributes = new Attributes(node);
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
CancellationHelper.ThrowIfCancelled();
}
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, NtfsFile ntfsFile)
{
if (parameters == null)
{
return;
}
if (ntfsFile == null)
{
return;
}
CancellationHelper.SetCancellationToken(parameters.CancelToken);
CancellationHelper.ThrowIfCancelled();
MFTNumber = ntfsFile.MFTRecord.MFTNumber;
SequenceNumber = ntfsFile.MFTRecord.SequenceNumber;
DriveLetter = driveLetter;
FileName = ntfsFile.Name;
MftTimeAccessed = ntfsFile.TimeAccessed;
MftTimeCreation = ntfsFile.TimeCreation;
MftTimeModified = ntfsFile.TimeModified;
MftTimeMftModified = ntfsFile.TimeMftModified;
DirectoryLocation = GetDirectory(ntfsFile);
Extension = Path.GetExtension(FileName);
FullPath = Path.Combine(DriveLetter.ToString() + DirectoryLocation, FileName);
this.Length = ntfsFile.AllocatedSize;
CancellationHelper.ThrowIfCancelled();
if (this.Length != 0)
{
// Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
if (this.Length >= (int.MaxValue - 1))
{
using (Stream ntfsStream = ntfsFile.OpenRead())
{
this.Sha256Hash = GetSha256Hash_BigFile(ntfsStream);
this.Length = ntfsStream.Length;
}
this.PeData = PeDataObject.TryGetPeDataObject(FileName, parameters.OnlineCertValidation);
CancellationHelper.ThrowIfCancelled();
this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FileName);
CancellationHelper.ThrowIfCancelled();
}
else
{
byte[] fileBytes = GetFileBytes(ntfsFile);
CancellationHelper.ThrowIfCancelled();
this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation);
if (IsPeDataPopulated)
{
this.Sha256Hash = PeData.SHA256Hash;
}
else
{
this.Sha256Hash = GetSha256Hash(fileBytes);
}
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
CancellationHelper.ThrowIfCancelled();
}
this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes);
CancellationHelper.ThrowIfCancelled();
if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
{
this.YaraRulesMatched = YaraRules.Scan(fileBytes, parameters.YaraRulesFilePath);
CancellationHelper.ThrowIfCancelled();
}
fileBytes = null;
}
}
PopulateFileInfoProperties(FullPath);
CancellationHelper.ThrowIfCancelled();
this.Attributes = new Attributes(FullPath);
CancellationHelper.ThrowIfCancelled();
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
IEnumerable <byte[]> fileChunks = null;
using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes))
{
fileChunks = node.GetBytes();
}
using (var timer = new TimingMetrics(TimingMetric.FileHashing))
{
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
}
using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties))
{
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
}
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
using (var timer = new TimingMetrics(TimingMetric.YaraScanning))
{
try
{
_yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex);
}
}
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy))
{
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
}
CancellationHelper.ThrowIfCancelled();
}
}
