コード例 #1
0
        public static ExtendedHashes FromFile(string file)
        {
            var hashes = new ExtendedHashes();

            using (var stream = File.OpenRead(file))
            {
                hashes.SHA256   = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
                stream.Position = 0;
                hashes.SHA1     = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
                stream.Position = 0;
                hashes.MD5      = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
                stream.Position = 0;

                var bytes = new byte[1024 * 8];
                var crc   = new Crc32();
                while (true)
                {
                    var read = stream.Read(bytes, 0, bytes.Length);
                    if (read == 0)
                    {
                        break;
                    }
                    crc.Update(bytes, 0, read);
                }

                hashes.CRC = crc.DigestBytes().ToHex();
            }

            return(hashes);
        }
コード例 #2
0
ファイル: VirtualFile.cs プロジェクト: bryce0110/wabbajack
        public static async ValueTask <ExtendedHashes> FromFile(IExtractedFile file)
        {
            var hashes = new ExtendedHashes();

            await using var stream = await file.OpenRead();

            hashes.SHA256   = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
            stream.Position = 0;
            hashes.SHA1     = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
            stream.Position = 0;
            hashes.MD5      = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
            stream.Position = 0;

            var bytes = new byte[1024 * 8];
            var crc   = new Crc32();

            while (true)
            {
                var read = stream.Read(bytes, 0, bytes.Length);
                if (read == 0)
                {
                    break;
                }
                crc.Update(bytes, 0, read);
            }

            hashes.CRC = crc.DigestBytes().ToHex();

            return(hashes);
        }
コード例 #3
0
ファイル: VirtualFile.cs プロジェクト: bryce0110/wabbajack
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            var hash = await extractedFile.HashAsync();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {relPath.FileName}");


                    return(ConvertFromIndexedFile(context, result, relPath, parent, extractedFile));
                }
            }

            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
            {
                return(vself);
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = extractedFile.Size,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
            }

            if (!await extractedFile.CanExtract())
            {
                return(self);
            }

            try
            {
                await using var extracted = await extractedFile.ExtractAll(context.Queue);

                var list = await extracted
                           .PMap(context.Queue,
                                 file => Analyze(context, self, file.Value, file.Key, depth + 1));

                self.Children = list.ToImmutableList();
            }
            catch (Exception ex)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            await using var ms = new MemoryStream();
            self.ToIndexedVirtualFile().ToJson(ms);
            _vfsCache.Put(self.Hash.ToArray(), ms.ToArray());

            return(self);
        }
コード例 #4
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
                                                       IPath relPath, int depth = 0)
        {
            var hash = absPath.FileHash();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {(string)absPath}");

                    VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
                    {
                        var vself = new VirtualFile
                        {
                            Context      = context,
                            Name         = path,
                            Parent       = vparent,
                            Size         = file.Size,
                            LastModified = absPath.LastModifiedUtc.AsUnixTime(),
                            LastAnalyzed = DateTime.Now.AsUnixTime(),
                            Hash         = file.Hash
                        };

                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();

                        return(vself);
                    }

                    return(Convert(result, relPath, parent));
                }
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = absPath.Size,
                LastModified = absPath.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
            }

            if (await FileExtractor.CanExtract(absPath))
            {
                await using var tempFolder = Context.GetTemporaryFolder();
                await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);

                var list = await tempFolder.FullName.EnumerateFiles()
                           .PMap(context.Queue,
                                 absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));

                self.Children = list.ToImmutableList();
            }

            return(self);
        }
コード例 #5
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
                                                       string rel_path, bool topLevel)
        {
            var hash = abs_path.FileHash();
            var fi   = new FileInfo(abs_path);

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}");
                    VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent)
                    {
                        var vself = new VirtualFile
                        {
                            Context      = context,
                            Name         = path,
                            Parent       = vparent,
                            Size         = file.Size,
                            LastModified = fi.LastWriteTimeUtc.Ticks,
                            LastAnalyzed = DateTime.Now.Ticks,
                            Hash         = file.Hash,
                        };

                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();

                        return(vself);
                    }

                    return(Convert(result, rel_path, parent));
                }
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = rel_path,
                Parent       = parent,
                Size         = fi.Length,
                LastModified = fi.LastWriteTimeUtc.Ticks,
                LastAnalyzed = DateTime.Now.Ticks,
                Hash         = hash
            };

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
            }

            if (FileExtractor.CanExtract(abs_path))
            {
                using (var tempFolder = context.GetTemporaryFolder())
                {
                    await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);

                    var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
                               .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false));

                    self.Children = list.ToImmutableList();
                }
            }

            return(self);
        }
コード例 #6
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            Hash hash;

            if (extractedFile is NativeFileStreamFactory)
            {
                hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty;
            }
            else
            {
                await using var hstream = await extractedFile.GetStream();

                hash = await hstream.xxHashAsync();
            }

            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
            {
                return(vself);
            }


            await using var stream = await extractedFile.GetStream();

            var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);

            stream.Position = 0;

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = stream.Length,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash,
            };

            if (Consts.TextureExtensions.Contains(relPath.FileName.Extension))
            {
                self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false);
            }

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
            }

            // Can't extract, so return
            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
            {
                await WriteToCache(self);

                return(self);
            }

            try
            {
                var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
                                                                 _ => true,
                                                                 async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));

                self.Children = list.Values.ToImmutableList();
            }
            catch (EndOfStreamException)
            {
                return(self);
            }
            catch (Exception)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            await WriteToCache(self);

            return(self);
        }
コード例 #7
0
ファイル: VirtualFile.cs プロジェクト: LordOfRhun/wabbajack
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            var hash = await extractedFile.HashAsync();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {relPath.FileName}");

                    VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
                    {
                        var vself = new VirtualFile
                        {
                            Context      = context,
                            Name         = path,
                            Parent       = vparent,
                            Size         = file.Size,
                            LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                            LastAnalyzed = DateTime.Now.AsUnixTime(),
                            Hash         = file.Hash
                        };

                        vself.FillFullPath();

                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();

                        return(vself);
                    }

                    return(Convert(result, relPath, parent));
                }
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = extractedFile.Size,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
            }

            if (!await extractedFile.CanExtract())
            {
                return(self);
            }

            try
            {
                await using var extracted = await extractedFile.ExtractAll(context.Queue);

                var list = await extracted
                           .PMap(context.Queue,
                                 file => Analyze(context, self, file.Value, file.Key, depth + 1));

                self.Children = list.ToImmutableList();
            }
            catch (Exception ex)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            return(self);
        }