private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
        {
            IEnumerable <byte[]> fileChunks = node.GetBytes();

            _timingMetrics.Start(TimingMetric.FileHashing);
            this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
            if (hasFileReadPermissions)
            {
                this._peData = PeDataObject.TryGetPeDataObject(FullPath);
                if (_peData != null)
                {
                    this.SHA1 = _peData.SHA1Hash;
                    this.MD5  = _peData.MD5Hash;
                }
                else
                {
                    this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
                    this.MD5  = Hash.ByteEnumerable.MD5(fileChunks);
                }
            }
            _timingMetrics.Stop(TimingMetric.FileHashing);

            if (_peData != null)
            {
                this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
            }

            CancellationHelper.ThrowIfCancelled();

            if (hasFileReadPermissions)
            {
                PopulateFileInfoProperties(FullPath);
                PopulateShellFileInfo(FullPath);
                CancellationHelper.ThrowIfCancelled();
            }

            if (hasFileReadPermissions)
            {
                Rules compiledRules = GetCompiledYaraRules(parameters.YaraParameters);
                if (compiledRules != null)
                {
                    _timingMetrics.Start(TimingMetric.YaraScanning);
                    this._yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
                    _timingMetrics.Stop(TimingMetric.YaraScanning);
                    compiledRules = null;
                }

                CancellationHelper.ThrowIfCancelled();
            }

            if (parameters.CalculateEntropy)             // Should we calculate entropy on really large files?
            {
                _timingMetrics.Start(TimingMetric.CalculatingEntropy);
                this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
                _timingMetrics.Stop(TimingMetric.CalculatingEntropy);
                CancellationHelper.ThrowIfCancelled();
            }
        }
Exemple #2
0
        private YSScanner GetCompiledYaraRules(FileEnumeratorParameters parameters)
        {
            YSScanner results = null;

            using (var timer = new TimingMetrics(TimingMetric.YaraRuleCompiling))
            {
                List <YaraFilter> yaraFilters = parameters.YaraParameters;

                List <string> distinctRulesToRun =
                    yaraFilters
                    .SelectMany(yf => yf.ProcessRule(this))
                    .Distinct()
                    .ToList();

                if (!distinctRulesToRun.Any())
                {
                    distinctRulesToRun =
                        yaraFilters
                        .Where(yf => yf.FilterType == YaraFilterType.ElseNoMatch)
                        .SelectMany(yf => yf.OnMatchRules)
                        .Distinct()
                        .ToList();
                }

                if (!distinctRulesToRun.Any())
                {
                    return(null);
                }

                distinctRulesToRun = distinctRulesToRun.OrderBy(s => s).ToList();

                string uniqueRuleCollectionToken = string.Join("|", distinctRulesToRun);
                string ruleCollectionHash        = Hash.ByteArray.Sha256(Encoding.UTF8.GetBytes(uniqueRuleCollectionToken));



                if (_yaraCompiledRulesDictionary.ContainsKey(ruleCollectionHash))
                {
                    results = _yaraCompiledRulesDictionary[ruleCollectionHash];
                }
                else
                {
                    try
                    {
                        results = YaraHelper.CompileRules(distinctRulesToRun, parameters.ReportAndLogOutputFunction);
                    }
                    catch (Exception ex)
                    {
                        parameters.ReportExceptionFunction.Invoke(nameof(GetCompiledYaraRules), string.Empty, ex);
                    }

                    _yaraCompiledRulesDictionary.Add(ruleCollectionHash, results);
                }
            }

            return(results);
        }
        public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node)
        {
            if (parameters == null)
            {
                return;
            }
            if (node == null)
            {
                return;
            }

            CancellationHelper.SetCancellationToken(parameters.CancelToken);
            CancellationHelper.ThrowIfCancelled();

            MFTNumber      = node.MFTRecordNumber;
            SequenceNumber = node.SequenceNumber;

            DriveLetter = driveLetter;
            FileName    = node.Name;

            MftTimeAccessed    = node.LastAccessTime;
            MftTimeCreation    = node.CreationTime;
            MftTimeModified    = node.LastChangeTime;
            MftTimeMftModified = node.TimeMftModified;

            DirectoryLocation = Path.GetDirectoryName(node.FullName);
            Extension         = Path.GetExtension(FileName);
            FullPath          = node.FullName;

            this.Length = node.Size;

            CancellationHelper.ThrowIfCancelled();

            if (this.Length == 0)             //Workaround for hard links
            {
                if (System.IO.File.Exists(FullPath))
                {
                    long length = new System.IO.FileInfo(FullPath).Length;
                    if (length > 0)
                    {
                        this.Length = (ulong)length;
                        node.Size   = this.Length;
                    }
                }
            }

            bool haveFileReadPermission = true;
            var  fileIOPermission       = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath);

            if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read)
            {
                haveFileReadPermission = true;
            }

            if (this.Length != 0)
            {
                var maxLength = ((ulong)NtfsReader.MaxClustersToRead * (ulong)4096);
                // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
                if (this.Length > maxLength)                 //(int.MaxValue - 1))
                {
                    IEnumerable <byte[]> fileChunks = node.GetBytes();
                    CancellationHelper.ThrowIfCancelled();

                    this.Sha256Hash = GetSha256Hash_IEnumerable(fileChunks, this.Length);
                    CancellationHelper.ThrowIfCancelled();

                    if (haveFileReadPermission)
                    {
                        this.PeData = PeDataObject.TryGetPeDataObject(FullPath, parameters.OnlineCertValidation);
                        CancellationHelper.ThrowIfCancelled();

                        this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FullPath);
                        CancellationHelper.ThrowIfCancelled();
                    }

                    /*
                     * if (parameters.CalculateEntropy)
                     * {
                     *      this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
                     *      CancellationHelper.ThrowIfCancelled();
                     * }
                     */

                    if (haveFileReadPermission)
                    {
                        if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
                        {
                            this.YaraRulesMatched = YaraRules.ScanFile(FullPath, parameters.YaraRulesFilePath);
                            CancellationHelper.ThrowIfCancelled();
                        }
                    }
                }
                else
                {
                    byte[] fileBytes = new byte[0];
                    if (!node.Streams.Any())                     //workaround for no file stream such as with hard links
                    {
                        try
                        {
                            using (FileStream fsSource = new FileStream(FullPath,
                                                                        FileMode.Open, FileAccess.Read))
                            {
                                // Read the source file into a byte array.
                                fileBytes = new byte[fsSource.Length];
                                int numBytesToRead = (int)fsSource.Length;
                                int numBytesRead   = 0;
                                while (numBytesToRead > 0)
                                {
                                    // Read may return anything from 0 to numBytesToRead.
                                    int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);

                                    // Break when the end of the file is reached.
                                    if (n == 0)
                                    {
                                        break;
                                    }

                                    numBytesRead   += n;
                                    numBytesToRead -= n;
                                }
                                numBytesToRead = fileBytes.Length;
                            }
                        }
                        catch (Exception ex)
                        {
                        }
                    }

                    else
                    {
                        fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
                        CancellationHelper.ThrowIfCancelled();
                    }
                    this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation);
                    if (IsPeDataPopulated)
                    {
                        this.Sha256Hash = PeData.SHA256Hash;
                    }
                    else
                    {
                        this.Sha256Hash = GetSha256Hash_Array(fileBytes);
                    }
                    CancellationHelper.ThrowIfCancelled();

                    this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes);
                    CancellationHelper.ThrowIfCancelled();

                    if (parameters.CalculateEntropy)
                    {
                        this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
                        CancellationHelper.ThrowIfCancelled();
                    }

                    if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
                    {
                        this.YaraRulesMatched = YaraRules.ScanBytes(fileBytes, parameters.YaraRulesFilePath);
                        CancellationHelper.ThrowIfCancelled();
                    }
                }
            }

            this.Attributes = new Attributes(node);
            CancellationHelper.ThrowIfCancelled();

            if (haveFileReadPermission)
            {
                PopulateFileInfoProperties(FullPath);
                CancellationHelper.ThrowIfCancelled();

                PopulateShellFileInfo(FullPath);
                CancellationHelper.ThrowIfCancelled();
            }
        }
        private void PopulateSmallFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
        {
            _timingMetrics.Start(TimingMetric.ReadingMFTBytes);
            byte[] fileBytes = new byte[0];
            if (!node.Streams.Any())             //workaround for no file stream such as with hard links
            {
                try
                {
                    using (FileStream fsSource = new FileStream(FullPath, FileMode.Open, FileAccess.Read))
                    {
                        // Read the source file into a byte array.
                        fileBytes = new byte[fsSource.Length];
                        int numBytesToRead = (int)fsSource.Length;
                        int numBytesRead   = 0;
                        while (numBytesToRead > 0)
                        {
                            // Read may return anything from 0 to numBytesToRead.
                            int n = fsSource.Read(fileBytes, numBytesRead, numBytesToRead);

                            // Break when the end of the file is reached.
                            if (n == 0)
                            {
                                break;
                            }

                            numBytesRead   += n;
                            numBytesToRead -= n;
                        }
                        numBytesToRead = fileBytes.Length;
                    }
                }
                catch (System.IO.IOException ioException)
                {
                    if (ioException.Message.Contains("contains a virus"))
                    {
                        string hash = "File access blocked by anti-virus program.";
                        this.Sha256 = hash;
                        this.SHA1   = hash;
                        this.MD5    = hash;
                        return;
                    }
                }
                catch
                { }
            }
            else
            {
                fileBytes = node.GetBytes().SelectMany(chunk => chunk).ToArray();
            }
            _timingMetrics.Stop(TimingMetric.ReadingMFTBytes);

            _timingMetrics.Start(TimingMetric.FileHashing);
            this._peData = PeDataObject.TryGetPeDataObject(fileBytes);
            if (this._peData != null)
            {
                this.Sha256 = _peData.SHA256Hash;
                this.SHA1   = _peData.SHA1Hash;
                this.MD5    = _peData.MD5Hash;
            }

            if (this._peData == null || string.IsNullOrWhiteSpace(this.Sha256))
            {
                this.Sha256 = Hash.ByteArray.Sha256(fileBytes);
                this.SHA1   = Hash.ByteArray.Sha1(fileBytes);
                this.MD5    = Hash.ByteArray.MD5(fileBytes);
            }
            _timingMetrics.Stop(TimingMetric.FileHashing);

            _timingMetrics.Start(TimingMetric.MiscFileProperties);
            if (_peData != null)
            {
                _authenticode = AuthenticodeData.GetAuthenticodeData(_peData.Certificate);
            }

            CancellationHelper.ThrowIfCancelled();

            if (hasFileReadPermissions)
            {
                PopulateFileInfoProperties(FullPath);
            }
            _timingMetrics.Stop(TimingMetric.MiscFileProperties);

            if (hasFileReadPermissions)
            {
                PopulateShellFileInfo(FullPath);
                CancellationHelper.ThrowIfCancelled();
            }

            YSScanner compiledRules = GetCompiledYaraRules(parameters);

            if (compiledRules != null)
            {
                _timingMetrics.Start(TimingMetric.YaraScanning);
                try
                {
                    _yaraRulesMatched = YaraHelper.ScanBytes(fileBytes, compiledRules);
                }
                catch (Exception ex)
                {
                    parameters.ReportExceptionFunction.Invoke(nameof(PopulateSmallFile), string.Empty, ex);
                }
                _timingMetrics.Stop(TimingMetric.YaraScanning);
            }
            CancellationHelper.ThrowIfCancelled();

            if (parameters.CalculateEntropy)
            {
                _timingMetrics.Start(TimingMetric.CalculatingEntropy);
                Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
                _timingMetrics.Stop(TimingMetric.CalculatingEntropy);
                CancellationHelper.ThrowIfCancelled();
            }
        }
        public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, INode node, TimingMetrics timingMetrics)
        {
            if (parameters == null)
            {
                return;
            }
            if (node == null)
            {
                return;
            }

            _timingMetrics = timingMetrics;

            CancellationHelper.SetCancellationToken(parameters.CancelToken);
            CancellationHelper.ThrowIfCancelled();

            _timingMetrics.Start(TimingMetric.MiscFileProperties);

            MFTNumber      = node.MFTRecordNumber;
            SequenceNumber = node.SequenceNumber;

            DriveLetter = driveLetter;
            FileName    = node.Name;

            MftTimeAccessed    = node.LastAccessTime;
            MftTimeCreation    = node.CreationTime;
            MftTimeModified    = node.LastChangeTime;
            MftTimeMftModified = node.TimeMftModified;

            DirectoryLocation = Path.GetDirectoryName(node.FullName);
            Extension         = Path.GetExtension(FileName);
            FullPath          = node.FullName;

            this.Length = node.Size;

            CancellationHelper.ThrowIfCancelled();

            if (this.Length == 0)             //Workaround for hard links
            {
                if (File.Exists(FullPath))
                {
                    long length = new FileInfo(FullPath).Length;
                    if (length > 0)
                    {
                        this.Length = (ulong)length;
                        node.Size   = this.Length;
                    }
                }
            }

            bool hasFileReadPermissions = true;
            var  fileIOPermission       = new FileIOPermission(FileIOPermissionAccess.Read, AccessControlActions.View, FullPath);

            if (fileIOPermission.AllFiles == FileIOPermissionAccess.Read)
            {
                hasFileReadPermissions = true;
            }

            _timingMetrics.Stop(TimingMetric.MiscFileProperties);

            if (this.Length != 0)
            {
                // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
                if (this.Length > MaxLength)                 //(int.MaxValue - 1))
                {
                    PopulateLargeFile(parameters, node, hasFileReadPermissions);
                }
                else
                {
                    PopulateSmallFile(parameters, node, hasFileReadPermissions);
                }
            }
            else
            {
                this.Sha256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855";                 // SHA256 hash of a null or zero-length input
                this.SHA1   = "DA39A3EE5E6B4B0D3255BFEF95601890AFD80709";
                this.MD5    = "D41D8CD98F00B204E9800998ECF8427E";
            }

            _timingMetrics.Start(TimingMetric.MiscFileProperties);

            PopulateIsTrusted();
            CancellationHelper.ThrowIfCancelled();

            this._attributes = new Attributes(node);
            _timingMetrics.Stop(TimingMetric.MiscFileProperties);
            CancellationHelper.ThrowIfCancelled();
        }
Exemple #6
0
        public void PopulateFileProperties(FileEnumeratorParameters parameters, char driveLetter, NtfsFile ntfsFile)
        {
            if (parameters == null)
            {
                return;
            }
            if (ntfsFile == null)
            {
                return;
            }

            CancellationHelper.SetCancellationToken(parameters.CancelToken);
            CancellationHelper.ThrowIfCancelled();

            MFTNumber      = ntfsFile.MFTRecord.MFTNumber;
            SequenceNumber = ntfsFile.MFTRecord.SequenceNumber;

            DriveLetter = driveLetter;
            FileName    = ntfsFile.Name;

            MftTimeAccessed    = ntfsFile.TimeAccessed;
            MftTimeCreation    = ntfsFile.TimeCreation;
            MftTimeModified    = ntfsFile.TimeModified;
            MftTimeMftModified = ntfsFile.TimeMftModified;

            DirectoryLocation = GetDirectory(ntfsFile);
            Extension         = Path.GetExtension(FileName);
            FullPath          = Path.Combine(DriveLetter.ToString() + DirectoryLocation, FileName);

            this.Length = ntfsFile.AllocatedSize;

            CancellationHelper.ThrowIfCancelled();

            if (this.Length != 0)
            {
                // Some entries in the MFT are greater than int.MaxValue !! That or the size is corrupt. Either way, we handle that here.
                if (this.Length >= (int.MaxValue - 1))
                {
                    using (Stream ntfsStream = ntfsFile.OpenRead())
                    {
                        this.Sha256Hash = GetSha256Hash_BigFile(ntfsStream);
                        this.Length     = ntfsStream.Length;
                    }

                    this.PeData = PeDataObject.TryGetPeDataObject(FileName, parameters.OnlineCertValidation);
                    CancellationHelper.ThrowIfCancelled();
                    this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(FileName);
                    CancellationHelper.ThrowIfCancelled();
                }
                else
                {
                    byte[] fileBytes = GetFileBytes(ntfsFile);

                    CancellationHelper.ThrowIfCancelled();

                    this.PeData = PeDataObject.TryGetPeDataObject(fileBytes, parameters.OnlineCertValidation);
                    if (IsPeDataPopulated)
                    {
                        this.Sha256Hash = PeData.SHA256Hash;
                    }
                    else
                    {
                        this.Sha256Hash = GetSha256Hash(fileBytes);
                    }

                    CancellationHelper.ThrowIfCancelled();

                    if (parameters.CalculateEntropy)
                    {
                        this.Entropy = EntropyHelper.CalculateFileEntropy(fileBytes);
                        CancellationHelper.ThrowIfCancelled();
                    }

                    this.Authenticode = AuthenticodeData.TryGetAuthenticodeData(fileBytes);

                    CancellationHelper.ThrowIfCancelled();

                    if (!string.IsNullOrWhiteSpace(parameters.YaraRulesFilePath) && File.Exists(parameters.YaraRulesFilePath))
                    {
                        this.YaraRulesMatched = YaraRules.Scan(fileBytes, parameters.YaraRulesFilePath);
                        CancellationHelper.ThrowIfCancelled();
                    }

                    fileBytes = null;
                }
            }

            PopulateFileInfoProperties(FullPath);

            CancellationHelper.ThrowIfCancelled();

            this.Attributes = new Attributes(FullPath);

            CancellationHelper.ThrowIfCancelled();

            PopulateShellFileInfo(FullPath);

            CancellationHelper.ThrowIfCancelled();
        }
Exemple #7
0
        private void PopulateLargeFile(FileEnumeratorParameters parameters, INode node, bool hasFileReadPermissions)
        {
            IEnumerable <byte[]> fileChunks = null;

            using (var timer = new TimingMetrics(TimingMetric.ReadingMFTBytes))
            {
                fileChunks = node.GetBytes();
            }

            using (var timer = new TimingMetrics(TimingMetric.FileHashing))
            {
                this.Sha256 = Hash.ByteEnumerable.Sha256(fileChunks);
                if (hasFileReadPermissions)
                {
                    this._peData = PeDataObject.TryGetPeDataObject(FullPath);
                    if (_peData != null)
                    {
                        this.SHA1 = _peData.SHA1Hash;
                        this.MD5  = _peData.MD5Hash;
                    }
                    else
                    {
                        this.SHA1 = Hash.ByteEnumerable.Sha1(fileChunks);
                        this.MD5  = Hash.ByteEnumerable.MD5(fileChunks);
                    }
                }
            }

            using (var timer = new TimingMetrics(TimingMetric.MiscFileProperties))
            {
                if (_peData != null)
                {
                    this._authenticode = AuthenticodeData.GetAuthenticodeData(this._peData.Certificate);
                }

                CancellationHelper.ThrowIfCancelled();

                if (hasFileReadPermissions)
                {
                    PopulateFileInfoProperties(FullPath);
                }
            }

            if (hasFileReadPermissions)
            {
                PopulateShellFileInfo(FullPath);
                CancellationHelper.ThrowIfCancelled();
            }

            if (hasFileReadPermissions)
            {
                YSScanner compiledRules = GetCompiledYaraRules(parameters);
                if (compiledRules != null)
                {
                    using (var timer = new TimingMetrics(TimingMetric.YaraScanning))
                    {
                        try
                        {
                            _yaraRulesMatched = YaraHelper.ScanFile(FullPath, compiledRules);
                        }
                        catch (Exception ex)
                        {
                            parameters.ReportExceptionFunction.Invoke(nameof(PopulateLargeFile), string.Empty, ex);
                        }
                    }
                }

                CancellationHelper.ThrowIfCancelled();
            }

            if (parameters.CalculateEntropy)             // Should we calculate entropy on really large files?
            {
                using (var timer = new TimingMetrics(TimingMetric.CalculatingEntropy))
                {
                    this.Entropy = EntropyHelper.CalculateFileEntropy(fileChunks, this.Length);
                }
                CancellationHelper.ThrowIfCancelled();
            }
        }