public static async ValueTask <ExtendedHashes> FromFile(IExtractedFile file) { var hashes = new ExtendedHashes(); await using var stream = await file.OpenRead(); hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex(); stream.Position = 0; hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex(); stream.Position = 0; hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex(); stream.Position = 0; var bytes = new byte[1024 * 8]; var crc = new Crc32(); while (true) { var read = stream.Read(bytes, 0, bytes.Length); if (read == 0) { break; } crc.Update(bytes, 0, read); } hashes.CRC = crc.DigestBytes().ToHex(); return(hashes); }
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile, IPath relPath, int depth = 0) { var hash = await extractedFile.HashAsync(); if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension)) { var result = await TryGetContentsFromServer(hash); if (result != null) { Utils.Log($"Downloaded VFS data for {relPath.FileName}"); return(ConvertFromIndexedFile(context, result, relPath, parent, extractedFile)); } } if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself)) { return(vself); } var self = new VirtualFile { Context = context, Name = relPath, Parent = parent, Size = extractedFile.Size, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash }; self.FillFullPath(depth); if (context.UseExtendedHashes) { self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile); } if (!await extractedFile.CanExtract()) { return(self); } try { await using var extracted = await extractedFile.ExtractAll(context.Queue); var list = await extracted .PMap(context.Queue, file => Analyze(context, self, file.Value, file.Key, depth + 1)); self.Children = list.ToImmutableList(); } catch (Exception ex) { Utils.Log($"Error while examining the contents of {relPath.FileName}"); throw; } await using var ms = new MemoryStream(); self.ToIndexedVirtualFile().ToJson(ms); _vfsCache.Put(self.Hash.ToArray(), ms.ToArray()); return(self); }
private static VirtualFile ConvertFromIndexedFile(Context context, IndexedVirtualFile file, IPath path, VirtualFile vparent, IExtractedFile extractedFile) { var vself = new VirtualFile { Context = context, Name = path, Parent = vparent, Size = file.Size, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = file.Hash }; vself.FillFullPath(); vself.Children = file.Children.Select(f => ConvertFromIndexedFile(context, f, f.Name, vself, extractedFile)).ToImmutableList(); return(vself); }
private static bool TryGetFromCache(Context context, VirtualFile parent, IPath path, IExtractedFile extractedFile, Hash hash, out VirtualFile found) { var result = _vfsCache.Get(hash.ToArray()); if (result == null) { found = null; return(false); } var data = new MemoryStream(result).FromJson <IndexedVirtualFile>(); found = ConvertFromIndexedFile(context, data, path, parent, extractedFile); return(true); }
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile, IPath relPath, int depth = 0) { var hash = await extractedFile.HashAsync(); if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension)) { var result = await TryGetContentsFromServer(hash); if (result != null) { Utils.Log($"Downloaded VFS data for {relPath.FileName}"); VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent) { var vself = new VirtualFile { Context = context, Name = path, Parent = vparent, Size = file.Size, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = file.Hash }; vself.FillFullPath(); vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList(); return(vself); } return(Convert(result, relPath, parent)); } } var self = new VirtualFile { Context = context, Name = relPath, Parent = parent, Size = extractedFile.Size, LastModified = extractedFile.LastModifiedUtc.AsUnixTime(), LastAnalyzed = DateTime.Now.AsUnixTime(), Hash = hash }; self.FillFullPath(depth); if (context.UseExtendedHashes) { self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile); } if (!await extractedFile.CanExtract()) { return(self); } try { await using var extracted = await extractedFile.ExtractAll(context.Queue); var list = await extracted .PMap(context.Queue, file => Analyze(context, self, file.Value, file.Key, depth + 1)); self.Children = list.ToImmutableList(); } catch (Exception ex) { Utils.Log($"Error while examining the contents of {relPath.FileName}"); throw; } return(self); }