private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters)
{
YSScanner results = null;
using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling))
{
List <YaraFilter> yaraFilters = parameters.YaraParameters;
List <string> distinctRulesToRun =
yaraFilters
.SelectMany(yf => yf.ProcessRule(this))
.Distinct()
.ToList();
if (!distinctRulesToRun.Any())
{
distinctRulesToRun =
yaraFilters
.Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
.SelectMany(yf => yf.OnMatchRules)
.Distinct()
.ToList();
}
if (!distinctRulesToRun.Any())
{
return(null);
}
distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList();
string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun);
string ruleCollectionHash = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken));
if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash))
{
results = _yaraCompiledRulesDictionary[ruleCollectionHash];
}
else
{
try
{
results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex);
}
_yaraCompiledRulesDictionary.Add(ruleCollectionHash, results);
}
}
return(results);
}
private void PopulateShellFileInfo(string fullPath)
{
try
{
if (!ShellFile.IsPlatformSupported)
{
return;
}
using (var timer = new TimingMetrics(TimingMetric.GettingShellInfo))
{
using (ShellFile file = ShellFile.FromFilePath(fullPath))
{
using (ShellProperties fileProperties = file.Properties)
{
ShellProperties.PropertySystem shellProperty = fileProperties.System;
Project = shellProperty.Project.Value ?? "";
ProviderItemID = shellProperty.ProviderItemID.Value ?? "";
OriginalFileName = shellProperty.OriginalFileName.Value ?? "";
FileOwner = shellProperty.FileOwner.Value ?? "";
FileVersion = shellProperty.FileVersion.Value ?? "";
FileDescription = shellProperty.FileDescription.Value ?? "";
Trademarks = shellProperty.Trademarks.Value ?? "";
Link = shellProperty.Link.TargetUrl.Value ?? "";
Copyright = shellProperty.Copyright.Value ?? "";
Company = shellProperty.Company.Value ?? "";
ApplicationName = shellProperty.ApplicationName.Value ?? "";
Comment = shellProperty.Comment.Value ?? "";
Title = shellProperty.Title.Value ?? "";
CancellationHelper.ThrowIfCancelled();
MimeType = shellProperty.ContentType.Value ?? "";
InternalName = shellProperty.InternalName.Value ?? "";
ProductName = shellProperty.Software.ProductName.Value ?? "";
Language = shellProperty.Language.Value ?? "";
ComputerName = shellProperty.ComputerName.Value ?? "";
}
}
}
}
catch
{ }
}
public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node, TimingMetrics timingMetrics)
{
if (parameters == null)
{
return;
}
if (node == null)
{
return;
}
_timingMetrics = timingMetrics;
CancellationHelper.SetCancellationToken(parameters.CancelToken);
CancellationHelper.ThrowIfCancelled();
_timingMetrics.Start(TimingMetric.MiscFileProperties);
MFTNumber = node.MFTRecordNumber;
SequenceNumber = node.SequenceNumber;
DriveLetter = driveLetter;
FileName = node.Name;
MftTimeAccessed = node.LastAccessTime;
MftTimeCreation = node.CreationTime;
MftTimeModified = node.LastChangeTime;
MftTimeMftModified = node.TimeMftModified;
DirectoryLocation = Path.GetDirectoryName(node.FullName);
Extension = Path.GetExtension(FileName);
FullPath = node.FullName;
this.Length = node.Size;
CancellationHelper.ThrowIfCancelled();
if (this.Length == 0) //Workaround for hard links
{
if (File.Exists(FullPath))
{
long length = new FileInfo(FullPath).Length;
if (length > 0)
{
this.Length = (ulong)length;
node.Size = this.Length;
}
}
}
bool hasFileReadPermissions = true;
var fileIOPermission = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath);
if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read)
{
hasFileReadPermissions = true;
}
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
if (this.Length != 0)
{
// Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
if (this.Length > MaxLength) //(int.MaxValue - 1))
{
PopulateLargeFile(parameters, node, hasFileReadPermissions);
}
else
{
PopulateSmallFile(parameters, node, hasFileReadPermissions);
}
}
else
{
this.Sha256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"; // SHA256 hash of a null or zero-length input
this.SHA1 = "DA39A3EE5E6B4B0D3255BFEF95601890AFD80709";
this.MD5 = "D41D8CD98F00B204E9800998ECF8427E";
}
_timingMetrics.Start(TimingMetric.MiscFileProperties);
PopulateIsTrusted();
CancellationHelper.ThrowIfCancelled();
this._attributes = new Attributes(node);
_timingMetrics.Stop(TimingMetric.MiscFileProperties);
CancellationHelper.ThrowIfCancelled();
}
private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
byte[] fileBytes = new byte[0];
using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes))
{
if (!node.Streams.Any()) //workaround for no file stream such as with hard links
{
try
{
using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read))
{
// Read the source file into a byte array.
fileBytes = new byte[fsSource.Length];
int numBytesToRead = (int)fsSource.Length;
int numBytesRead = 0;
while (numBytesToRead > 0)
{
// Read may return anything from 0 to numBytesToRead.
int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);
// Break when the end of the file is reached.
if (n == 0)
{
break;
}
numBytesRead += n;
numBytesToRead -= n;
}
numBytesToRead = fileBytes.Length;
}
}
catch (System.IO.IOException ioException)
{
if (ioException.Message.Contains("contains a virus"))
{
string hash = "File access blocked by anti-virus program.";
this.Sha256 = hash;
this.SHA1 = hash;
this.MD5 = hash;
return;
}
}
catch
{ }
}
else
{
fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
}
}
using (var timer = new TimingMetrics(TimingMetric.FileHashing))
{
this._peData = PeDataObject.TryGetPeDataObject(fileBytes);
if (this._peData != null)
{
this.Sha256 = _peData.SHA256Hash;
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256))
{
this.Sha256 = Hash.ByteArray.Sha256(fileBytes);
this.SHA1 = Hash.ByteArray.Sha1(fileBytes);
this.MD5 = Hash.ByteArray.MD5(fileBytes);
}
}
using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties))
{
if (_peData != null)
{
_authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
}
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
using (var timer = new TimingMetrics(TimingMetric.YaraScanning))
{
try
{
_yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex);
}
}
}
CancellationHelper.ThrowIfCancelled();
if (parameters.CalculateEntropy)
{
using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy))
{
Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
}
CancellationHelper.ThrowIfCancelled();
}
}
private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
{
IEnumerable <byte[]> fileChunks = null;
using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes))
{
fileChunks = node.GetBytes();
}
using (var timer = new TimingMetrics(TimingMetric.FileHashing))
{
this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
if (hasFileReadPermissions)
{
this._peData = PeDataObject.TryGetPeDataObject(FullPath);
if (_peData != null)
{
this.SHA1 = _peData.SHA1Hash;
this.MD5 = _peData.MD5Hash;
}
else
{
this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
this.MD5 = Hash.ByteEnumerable.MD5(fileChunks);
}
}
}
using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties))
{
if (_peData != null)
{
this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
}
CancellationHelper.ThrowIfCancelled();
if (hasFileReadPermissions)
{
PopulateFileInfoProperties(FullPath);
}
}
if (hasFileReadPermissions)
{
PopulateShellFileInfo(FullPath);
CancellationHelper.ThrowIfCancelled();
}
if (hasFileReadPermissions)
{
YSScanner compiledRules = GetCompiledYaraRules(parameters);
if (compiledRules != null)
{
using (var timer = new TimingMetrics(TimingMetric.YaraScanning))
{
try
{
_yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
}
catch (Exception ex)
{
parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex);
}
}
}
CancellationHelper.ThrowIfCancelled();
}
if (parameters.CalculateEntropy) // Should we calculate entropy on really large files?
{
using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy))
{
this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
}
CancellationHelper.ThrowIfCancelled();
}
}
