public static ExtendedHashes FromFile(string file)
{
var hashes = new ExtendedHashes();
using (var stream = File.OpenRead(file))
{
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
var bytes = new byte[1024 * 8];
var crc = new Crc32();
while (true)
{
var read = stream.Read(bytes, 0, bytes.Length);
if (read == 0)
{
break;
}
crc.Update(bytes, 0, read);
}
hashes.CRC = crc.DigestBytes().ToHex();
}
return(hashes);
}
public static async ValueTask <ExtendedHashes> FromFile(IExtractedFile file)
{
var hashes = new ExtendedHashes();
await using var stream = await file.OpenRead();
hashes.SHA256 = System.Security.Cryptography.SHA256.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.SHA1 = System.Security.Cryptography.SHA1.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
hashes.MD5 = System.Security.Cryptography.MD5.Create().ComputeHash(stream).ToHex();
stream.Position = 0;
var bytes = new byte[1024 * 8];
var crc = new Crc32();
while (true)
{
var read = stream.Read(bytes, 0, bytes.Length);
if (read == 0)
{
break;
}
crc.Update(bytes, 0, read);
}
hashes.CRC = crc.DigestBytes().ToHex();
return(hashes);
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
IPath relPath, int depth = 0)
{
var hash = await extractedFile.HashAsync();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
return(ConvertFromIndexedFile(context, result, relPath, parent, extractedFile));
}
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
{
return(vself);
}
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = extractedFile.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = await ExtendedHashes.FromFile(extractedFile);
}
if (!await extractedFile.CanExtract())
{
return(self);
}
try
{
await using var extracted = await extractedFile.ExtractAll(context.Queue);
var list = await extracted
.PMap(context.Queue,
file => Analyze(context, self, file.Value, file.Key, depth + 1));
self.Children = list.ToImmutableList();
}
catch (Exception ex)
{
Utils.Log($"Error while examining the contents of {relPath.FileName}");
throw;
}
await using var ms = new MemoryStream();
self.ToIndexedVirtualFile().ToJson(ms);
_vfsCache.Put(self.Hash.ToArray(), ms.ToArray());
return(self);
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, AbsolutePath absPath,
IPath relPath, int depth = 0)
{
var hash = absPath.FileHash();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(absPath))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {(string)absPath}");
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
return(vself);
}
return(Convert(result, relPath, parent));
}
}
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = absPath.Size,
LastModified = absPath.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = ExtendedHashes.FromFile(absPath);
}
if (await FileExtractor.CanExtract(absPath))
{
await using var tempFolder = Context.GetTemporaryFolder();
await FileExtractor.ExtractAll(context.Queue, absPath, tempFolder.FullName);
var list = await tempFolder.FullName.EnumerateFiles()
.PMap(context.Queue,
absSrc => Analyze(context, self, absSrc, absSrc.RelativeTo(tempFolder.FullName), depth + 1));
self.Children = list.ToImmutableList();
}
return(self);
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, string abs_path,
string rel_path, bool topLevel)
{
var hash = abs_path.FileHash();
var fi = new FileInfo(abs_path);
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(abs_path))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {Path.GetFileName(abs_path)}");
VirtualFile Convert(IndexedVirtualFile file, string path, VirtualFile vparent)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks,
Hash = file.Hash,
};
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
return(vself);
}
return(Convert(result, rel_path, parent));
}
}
var self = new VirtualFile
{
Context = context,
Name = rel_path,
Parent = parent,
Size = fi.Length,
LastModified = fi.LastWriteTimeUtc.Ticks,
LastAnalyzed = DateTime.Now.Ticks,
Hash = hash
};
if (context.UseExtendedHashes)
{
self.ExtendedHashes = ExtendedHashes.FromFile(abs_path);
}
if (FileExtractor.CanExtract(abs_path))
{
using (var tempFolder = context.GetTemporaryFolder())
{
await FileExtractor.ExtractAll(context.Queue, abs_path, tempFolder.FullName);
var list = await Directory.EnumerateFiles(tempFolder.FullName, "*", SearchOption.AllDirectories)
.PMap(context.Queue, abs_src => Analyze(context, self, abs_src, abs_src.RelativeTo(tempFolder.FullName), false));
self.Children = list.ToImmutableList();
}
}
return(self);
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IStreamFactory extractedFile,
IPath relPath, int depth = 0)
{
Hash hash;
if (extractedFile is NativeFileStreamFactory)
{
hash = await((AbsolutePath)extractedFile.Name).FileHashCachedAsync() ?? Hash.Empty;
}
else
{
await using var hstream = await extractedFile.GetStream();
hash = await hstream.xxHashAsync();
}
if (TryGetFromCache(context, parent, relPath, extractedFile, hash, out var vself))
{
return(vself);
}
await using var stream = await extractedFile.GetStream();
var sig = await FileExtractor2.ArchiveSigs.MatchesAsync(stream);
stream.Position = 0;
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = stream.Length,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash,
};
if (Consts.TextureExtensions.Contains(relPath.FileName.Extension))
{
self.ImageState = await ImageState.FromImageStream(stream, relPath.FileName.Extension, false);
}
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = await ExtendedHashes.FromStream(stream);
}
// Can't extract, so return
if (!sig.HasValue || !FileExtractor2.ExtractableExtensions.Contains(relPath.FileName.Extension))
{
await WriteToCache(self);
return(self);
}
try
{
var list = await FileExtractor2.GatheringExtract(context.Queue, extractedFile,
_ => true,
async (path, sfactory) => await Analyze(context, self, sfactory, path, depth + 1));
self.Children = list.Values.ToImmutableList();
}
catch (EndOfStreamException)
{
return(self);
}
catch (Exception)
{
Utils.Log($"Error while examining the contents of {relPath.FileName}");
throw;
}
await WriteToCache(self);
return(self);
}
public static async Task <VirtualFile> Analyze(Context context, VirtualFile parent, IExtractedFile extractedFile,
IPath relPath, int depth = 0)
{
var hash = await extractedFile.HashAsync();
if (!context.UseExtendedHashes && FileExtractor.MightBeArchive(relPath.FileName.Extension))
{
var result = await TryGetContentsFromServer(hash);
if (result != null)
{
Utils.Log($"Downloaded VFS data for {relPath.FileName}");
VirtualFile Convert(IndexedVirtualFile file, IPath path, VirtualFile vparent)
{
var vself = new VirtualFile
{
Context = context,
Name = path,
Parent = vparent,
Size = file.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = file.Hash
};
vself.FillFullPath();
vself.Children = file.Children.Select(f => Convert(f, f.Name, vself)).ToImmutableList();
return(vself);
}
return(Convert(result, relPath, parent));
}
}
var self = new VirtualFile
{
Context = context,
Name = relPath,
Parent = parent,
Size = extractedFile.Size,
LastModified = extractedFile.LastModifiedUtc.AsUnixTime(),
LastAnalyzed = DateTime.Now.AsUnixTime(),
Hash = hash
};
self.FillFullPath(depth);
if (context.UseExtendedHashes)
{
self.ExtendedHashes = ExtendedHashes.FromFile(extractedFile);
}
if (!await extractedFile.CanExtract())
{
return(self);
}
try
{
await using var extracted = await extractedFile.ExtractAll(context.Queue);
var list = await extracted
.PMap(context.Queue,
file => Analyze(context, self, file.Value, file.Key, depth + 1));
self.Children = list.ToImmutableList();
}
catch (Exception ex)
{
Utils.Log($"Error while examining the contents of {relPath.FileName}");
throw;
}
return(self);
}