public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path, string rel_path, bool topLevel) { var hash = abs_path.FileHash(); var fi = new FileInfo(abs_path); if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path)) { var result = await TryGetContentsFromServer(hash); if (result != null) { Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}"); VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent) { var vself = new VirtualFile { Context = context, Name = path, Parent = vparent, Size = file.Size, LastModified = fi.LastWriteTimeUtc.Ticks, LastAnalyzed = DateTime.Now.Ticks, Hash = file.Hash, }; vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList(); return(vself); } return(Convert(result, rel_path, parent)); } } var self = new VirtualFile { Context = context, Name = rel_path, Parent = parent, Size = fi.Length, LastModified = fi.LastWriteTimeUtc.Ticks, LastAnalyzed = DateTime.Now.Ticks, Hash = hash }; if (context.UseExtendedHashes) { self.ExtendedHashes = ExtendedHashes.FromFile(abs_path); } if (FileExtractor.CanExtract(abs_path)) { using (var tempFolder = context.GetTemporaryFolder()) { await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName); var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories) .PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false)); self.Children = list.ToImmutableList(); } } return(self); }
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IStreamFactory extractedFile, Hash hash, out VirtualFile found) { using var cmd = new SQLiteCommand(_conn); cmd.CommandText = @"SELECT Contents FROM VFSCache WHERE Hash = @hash"; cmd.Parameters.AddWithValue("@hash", (long)hash); using var rdr = cmd.ExecuteReader(); while (rdr.Read()) { var data = IndexedVirtualFile.Read(rdr.GetStream(0)); found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); found.Name = path; found.Hash = hash; return(true); } found = default; return(false); }
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile, IPath relPath, int depth = 0) { Hash hash; if (extractedFile is NativeFileStreamFactory) { hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty; } else { await using var hstream = await extractedFile.GetStream(); hash = await hstream.xxHashAsync(); } if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) { return(vself); } await using var stream = await extractedFile.GetStream(); var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream); stream.Position = 0; var self = new VirtualFile { Context = context, Name = relPath, Parent = parent, Size = stream.Length, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash }; self.FillFullPath(depth); if (context.UseExtendedHashes) { self.ExtendedHashes = await ExtendedHashes.FromStream(stream); } // Can't extract, so return if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension)) { return(self); } try { var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile, _ => true, async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1)); self.Children = list.Values.ToImmutableList(); } catch (EndOfStreamException) { return(self); } catch (Exception) { Utils.Log($"Error while examining the contents of {relPath.FileName}"); throw; } await using var ms = new MemoryStream(); self.ToIndexedVirtualFile().Write(ms); ms.Position = 0; await InsertIntoVFSCache(self.Hash, ms); return(self); }
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IStreamFactory extractedFile) { var vself = new VirtualFile { Context = context, Name = path, Parent = vparent, Size = file.Size, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = file.Hash }; vself.FillFullPath(); vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList(); return(vself); }