示例#1
0
        private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile)
        {
            var vself = new VirtualFile
            {
                Context      = context,
                Name         = path,
                Parent       = vparent,
                Size         = file.Size,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = file.Hash
            };

            vself.FillFullPath();

            vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList();

            return(vself);
        }
示例#2
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            var hash = await extractedFile.HashAsync();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {relPath.FileName}");


                    return(ConvertFromIndexedFile(context, result, relPath, parent, extractedFile));
                }
            }

            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
            {
                return(vself);
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = extractedFile.Size,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
            }

            if (!await extractedFile.CanExtract())
            {
                return(self);
            }

            try
            {
                await using var extracted = await extractedFile.ExtractAll(context.Queue);

                var list = await extracted
                           .PMap(context.Queue,
                                 file => Analyze(context, self, file.Value, file.Key, depth + 1));

                self.Children = list.ToImmutableList();
            }
            catch (Exception ex)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            await using var ms = new MemoryStream();
            self.ToIndexedVirtualFile().ToJson(ms);
            _vfsCache.Put(self.Hash.ToArray(), ms.ToArray());

            return(self);
        }
示例#3
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            Hash hash;

            if (extractedFile is NativeFileStreamFactory)
            {
                hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty;
            }
            else
            {
                await using var hstream = await extractedFile.GetStream();

                hash = await hstream.xxHashAsync();
            }

            if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
            {
                return(vself);
            }


            await using var stream = await extractedFile.GetStream();

            var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);

            stream.Position = 0;

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = stream.Length,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash,
            };

            if (Consts.TextureExtensions.Contains(relPath.FileName.Extension))
            {
                self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false);
            }

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
            }

            // Can't extract, so return
            if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
            {
                await WriteToCache(self);

                return(self);
            }

            try
            {
                var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
                                                                 _ => true,
                                                                 async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));

                self.Children = list.Values.ToImmutableList();
            }
            catch (EndOfStreamException)
            {
                return(self);
            }
            catch (Exception)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            await WriteToCache(self);

            return(self);
        }
示例#4
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
                                                       IPath relPath, int depth = 0)
        {
            var hash = absPath.FileHash();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {(string)absPath}");

                    VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
                    {
                        var vself = new VirtualFile
                        {
                            Context      = context,
                            Name         = path,
                            Parent       = vparent,
                            Size         = file.Size,
                            LastModified = absPath.LastModifiedUtc.AsUnixTime(),
                            LastAnalyzed = DateTime.Now.AsUnixTime(),
                            Hash         = file.Hash
                        };

                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();

                        return(vself);
                    }

                    return(Convert(result, relPath, parent));
                }
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = absPath.Size,
                LastModified = absPath.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
            }

            if (await FileExtractor.CanExtract(absPath))
            {
                await using var tempFolder = Context.GetTemporaryFolder();
                await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);

                var list = await tempFolder.FullName.EnumerateFiles()
                           .PMap(context.Queue,
                                 absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));

                self.Children = list.ToImmutableList();
            }

            return(self);
        }
示例#5
0
        public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
                                                       IPath relPath, int depth = 0)
        {
            var hash = await extractedFile.HashAsync();

            if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
            {
                var result = await TryGetContentsFromServer(hash);

                if (result != null)
                {
                    Utils.Log($"Downloaded VFS data for {relPath.FileName}");

                    VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
                    {
                        var vself = new VirtualFile
                        {
                            Context      = context,
                            Name         = path,
                            Parent       = vparent,
                            Size         = file.Size,
                            LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                            LastAnalyzed = DateTime.Now.AsUnixTime(),
                            Hash         = file.Hash
                        };

                        vself.FillFullPath();

                        vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();

                        return(vself);
                    }

                    return(Convert(result, relPath, parent));
                }
            }

            var self = new VirtualFile
            {
                Context      = context,
                Name         = relPath,
                Parent       = parent,
                Size         = extractedFile.Size,
                LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
                LastAnalyzed = DateTime.Now.AsUnixTime(),
                Hash         = hash
            };

            self.FillFullPath(depth);

            if (context.UseExtendedHashes)
            {
                self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
            }

            if (!await extractedFile.CanExtract())
            {
                return(self);
            }

            try
            {
                await using var extracted = await extractedFile.ExtractAll(context.Queue);

                var list = await extracted
                           .PMap(context.Queue,
                                 file => Analyze(context, self, file.Value, file.Key, depth + 1));

                self.Children = list.ToImmutableList();
            }
            catch (Exception ex)
            {
                Utils.Log($"Error while examining the contents of {relPath.FileName}");
                throw;
            }

            return(self);
        }