public FileSource(string Filename)
{
this.Filename = Filename;
using (FileStream fs = new FileStream(Filename, FileMode.Open))
{
_size = (uint)fs.Length;
_md5 = Utility.GetMd5(fs);
}
}
public PPSource(IReadFile subfile)
{
this.subfile = subfile;
using (Stream stream = GetStream())
{
_md5 = Utility.GetMd5(stream);
_size = (uint)stream.Position;
}
}
protected async Task GenerateHashes(IList <ISubfile> files, IProgress <int> progressPercentage)
{
Dictionary <string, List <PPSource> > ppSubfiles = new Dictionary <string, List <PPSource> >();
foreach (var file in files)
{
if (file.Source is PPSource ppSource)
{
if (!ppSubfiles.TryGetValue(ppSource.Subfile.ppPath, out var subfileList))
{
subfileList = new List <PPSource>();
ppSubfiles[ppSource.Subfile.ppPath] = subfileList;
}
subfileList.Add(ppSource);
}
}
int i = 0;
double total = ppSubfiles.Sum(x => x.Value.Count);
await ppSubfiles.ForEachAsync(4, async ppFile =>
{
await using var fileStream = new FileStream(ppFile.Key, FileMode.Open, FileAccess.Read);
foreach (var file in ppFile.Value)
{
using var rentedMemory = MemoryPool <byte> .Shared.Rent((int)file.Subfile.size);
await using var substream = file.Subfile.CreateReadStream(fileStream);
int totalRead = 0;
int read = -1;
while (read != 0)
{
read = await substream.ReadAsync(rentedMemory.Memory.Slice(totalRead, (int)file.Subfile.size - totalRead));
totalRead += read;
}
file.Md5 = Utility.GetMd5(rentedMemory.Memory.Span.Slice(0, (int)file.Subfile.size));
progressPercentage.Report((int)(Interlocked.Increment(ref i) * 100 / total));
}
});
}
static void VerifyArgs(string[] args)
{
string currentArg;
int argcounter = 1;
bool fullVerify = true;
while ((currentArg = args[argcounter++]).StartsWith("-"))
{
currentArg = currentArg.ToLower();
if (currentArg == "-fullverify")
{
fullVerify = args[argcounter++].ToLower() == "on";
}
else
{
HaltAndCatchFire($"Unknown command: \"{currentArg}\"", 1);
return;
}
}
string filename = args[--argcounter];
if (!File.Exists(filename))
{
HaltAndCatchFire($"Input file does not exist: {filename}", 1);
}
ExtendedArchive arc;
try
{
arc = new ExtendedArchive(filename);
}
catch (PpexException ex)
{
if (ex.ErrorCode == PpexException.PpexErrorCode.FileNotPPXArchive ||
ex.ErrorCode == PpexException.PpexErrorCode.IncorrectVersionNumber)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex.Message);
Console.ResetColor();
return;
}
else
{
throw;
}
}
Console.CursorVisible = false;
Action <int, string> UpdateProgress = (i, x) =>
{
Console.SetCursorPosition(0, Console.CursorTop);
string message = $"Verification {i}% complete [{x}]";
message = message.PadRight(Console.BufferWidth - 1);
Console.Write(message);
};
var orderedChunks = arc.Chunks.OrderBy(x => x.Offset).ToArray();
Console.WriteLine("Archive name: " + arc.Title);
Console.WriteLine("File count: " + arc.Files.Count);
long currentOffset = 0;
long totalOffset = orderedChunks.Sum(x => (long)x.CompressedLength);
HashSet <(Md5Hash, ulong, ulong)> verifiedHashes = new HashSet <(Md5Hash, ulong, ulong)>();
for (int i = 0; i < orderedChunks.Length; i++)
{
ExtendedArchiveChunk chunk = orderedChunks[i];
using var compressedBuffer = MemoryPool <byte> .Shared.Rent((int) chunk.CompressedLength);
var compressedMemory = compressedBuffer.Memory.Slice(0, (int)chunk.CompressedLength);
using (var rawStream = chunk.GetRawStream())
{
rawStream.Read(compressedMemory.Span);
}
if (chunk.CRC32 != CRC32.Compute(compressedMemory.Span))
{
HaltAndCatchFire($"Chunk hash mismatch; id: {chunk.ID}, offset: {chunk.Offset}", 8);
}
if (fullVerify)
{
using var uncompressedBuffer = MemoryPool <byte> .Shared.Rent((int) chunk.UncompressedLength);
var uncompressedMemory = uncompressedBuffer.Memory.Slice(0, (int)chunk.UncompressedLength);
chunk.CopyToMemory(uncompressedMemory);
foreach (var file in chunk.Files)
{
var expectedHash = (file.RawSource.Md5, file.RawSource.Offset, file.RawSource.Size);
if (verifiedHashes.Contains(expectedHash))
{
continue;
}
var actualMd5 =
Utility.GetMd5(uncompressedMemory.Span.Slice((int)file.RawSource.Offset,
(int)file.RawSource.Size));
if ((Md5Hash)file.RawSource.Md5 != (Md5Hash)actualMd5)
{
HaltAndCatchFire($"File md5 mismatch; chunk id: {chunk.ID}, archive: {file.ArchiveName}, file: {file.Name}", 9);
}
verifiedHashes.Add((actualMd5, file.RawSource.Offset, file.RawSource.Size));
}
verifiedHashes.Clear();
}
currentOffset += (long)chunk.CompressedLength;
//int percentage = (int)((float)(100 * i) / (float)orderedChunks.Length);
int percentage = (int)Math.Round((float)(100 * currentOffset) / (float)totalOffset);
UpdateProgress(percentage, $"{i + 1}/{orderedChunks.Length} : {ConvertToReadable(currentOffset)}/{ConvertToReadable(totalOffset)}");
}
Console.WriteLine();
Console.WriteLine($"\"{arc.Title}\" successfully verified.");
Console.CursorVisible = true;
}
public void CompressCallback(int id)
{
using ZstdCompressor compressor = new ZstdCompressor();
var completionSource = threadCompletionSources[id];
try
{
while (!QueuedChunks.IsCompleted)
{
if (QueuedChunks.TryTake(out var queuedChunk, 500))
{
var totalUncompressed = queuedChunk.Subfiles.Sum(x => (int)x.Size);
var upperBound = ZstdCompressor.GetUpperCompressionBound(totalUncompressed);
var uncompressedBuffer = MemoryPool <byte> .Shared.Rent(totalUncompressed);
int currentBufferIndex = 0;
List <FileReceipt> fileReceipts = new List <FileReceipt>();
foreach (var subfile in queuedChunk.Subfiles)
{
try
{
FileReceipt receipt;
if ((receipt = fileReceipts.Find(x => x.Md5 == subfile.Source.Md5)) != null)
{
receipt = FileReceipt.CreateDuplicate(receipt, subfile);
receipt.Filename = subfile.Name;
receipt.EmulatedName = subfile.Name;
receipt.ArchiveName = subfile.ArchiveName;
}
else if (subfile.RequestedConversion != null)
{
if (subfile.RequestedConversion.TargetEncoding != ArchiveFileType.OpusAudio)
{
throw new NotImplementedException("Only supports opus encoding at this time");
}
using var opusEncoder = new OpusEncoder();
using var inputStream = subfile.GetStream();
using var bufferStream =
new MemorySpanStream(uncompressedBuffer.Memory.Slice(currentBufferIndex));
opusEncoder.Encode(inputStream, bufferStream);
receipt = new FileReceipt
{
Md5 = Utility.GetMd5(bufferStream.SliceToCurrentPosition().Span),
Length = (ulong)bufferStream.Position,
Offset = (ulong)0,
Filename = opusEncoder.RealNameTransform(subfile.Name),
EmulatedName = subfile.Name,
Encoding = ArchiveFileType.OpusAudio,
ArchiveName = subfile.ArchiveName,
Subfile = subfile
};
currentBufferIndex += (int)receipt.Length;
}
else
{
using var inputStream = subfile.GetStream();
int totalRead = 0;
while (totalRead < (int)subfile.Size)
{
int read = inputStream.Read(
uncompressedBuffer.Memory.Span.Slice(currentBufferIndex,
(int)subfile.Size - totalRead));
totalRead += read;
}
receipt = new FileReceipt
{
Md5 = subfile.Source.Md5,
Length = subfile.Size,
Offset = (ulong)currentBufferIndex,
Filename = subfile.Name,
EmulatedName = subfile.Name,
Encoding = subfile.Type,
ArchiveName = subfile.ArchiveName,
Subfile = subfile
};
currentBufferIndex += (int)receipt.Length;
}
fileReceipts.Add(receipt);
}
catch (Exception ex)
{
throw new Exception($"Failed to compress file '{subfile.ArchiveName}/{subfile.Name}'", ex);
}
}
Memory <byte> uncompressedSpan = uncompressedBuffer.Memory.Slice(0, currentBufferIndex);
IMemoryOwner <byte> compressedBuffer;
Memory <byte> compressedMemory;
if (queuedChunk.Compression == ArchiveChunkCompression.Zstandard)
{
compressedBuffer = MemoryPool <byte> .Shared.Rent(upperBound);
compressor.CompressData(uncompressedSpan.Span,
compressedBuffer.Memory.Span, queuedChunk.CompressionLevel, out int compressedSize);
compressedMemory = compressedBuffer.Memory.Slice(0, compressedSize);
uncompressedBuffer.Dispose();
}
else
{
compressedBuffer = uncompressedBuffer;
compressedMemory = uncompressedSpan;
}
uint crc = CRC32.Compute(compressedMemory.Span);
var chunkReceipt = new ChunkReceipt
{
ID = queuedChunk.ID,
Compression = queuedChunk.Compression,
CRC = crc,
UncompressedSize = (ulong)uncompressedSpan.Length,
CompressedSize = (ulong)compressedMemory.Length,
FileReceipts = fileReceipts
};
ReadyChunks.Add(new FinishedChunk
{
UnderlyingBuffer = compressedBuffer,
Data = compressedMemory,
Receipt = chunkReceipt
});
threadProgress.Report(
$"Compressed chunk id:{queuedChunk.ID} ({fileReceipts.Count} files) ({Utility.GetBytesReadable((long)chunkReceipt.CompressedSize)} - {(double)chunkReceipt.CompressedSize / chunkReceipt.UncompressedSize:P} ratio)\r\n");
}
else
{
Thread.Sleep(50);
}
}
completionSource.SetResult(null);
}
catch (Exception ex)
{
completionSource.SetException(ex);
}
}
