static async Task <uint> CalcZipCrc32HandleAsync(FileBase src, CopyFileParams param, CancellationToken cancel)
{
ZipCrc32 srcCrc = new ZipCrc32();
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer))
{
while (true)
{
int readSize = await src.ReadAsync(buffer, cancel);
Debug.Assert(readSize <= buffer.Length);
if (readSize <= 0)
{
break;
}
ReadOnlyMemory <byte> sliced = buffer.Slice(0, readSize);
srcCrc.Append(sliced.Span);
}
}
return(srcCrc.Value);
}
void UpdateKeys(byte c)
{
Key0 = ZipCrc32.CalcCrc32ForZipEncryption(Key0, c);
Key1 += (byte)Key0;
Key1 = Key1 * 134775813 + 1;
Key2 = ZipCrc32.CalcCrc32ForZipEncryption(Key2, (byte)(Key1 >> 24));
}
public void ZipCrc32()
{
var underTestCrc32 = new ZipCrc32();
Assert.AreEqual(0x0, underTestCrc32.Value);
underTestCrc32.Update(check);
Assert.AreEqual(0xCBF43926, underTestCrc32.Value);
underTestCrc32.Reset();
Assert.AreEqual(0x0, underTestCrc32.Value);
exceptionTesting(underTestCrc32);
}
bool ReadHeader()
{
// Initialize CRC for this block
crc = new ZipCrc32();
// Make sure there is data in file. We can't rely on ReadLeByte() to fill the buffer, as this could be EOF,
// which is fine, but ReadLeByte() throws an exception if it doesn't find data, so we do this part ourselves.
if (InputBuffer.Available <= 0)
{
InputBuffer.Fill();
if (InputBuffer.Available <= 0)
{
// No header, EOF.
return(false);
}
}
// 1. Check the two magic bytes
var headCRC = new ZipCrc32();
var magic = InputBuffer.ReadLeByte();
if (magic < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(magic);
if (magic != (GZipConstants.MAGIC >> 8))
{
throw new InvalidDataException("Error GZIP header, first magic byte doesn't match");
}
// magic = baseInputStream.ReadByte();
magic = InputBuffer.ReadLeByte();
if (magic < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
if (magic != (GZipConstants.MAGIC & 0xFF))
{
throw new InvalidDataException("Error GZIP header, second magic byte doesn't match");
}
headCRC.Update(magic);
// 2. Check the compression type (must be 8)
var compressionType = InputBuffer.ReadLeByte();
if (compressionType < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
if (compressionType != 8)
{
throw new InvalidDataException("Error GZIP header, data not in deflate format");
}
headCRC.Update(compressionType);
// 3. Check the flags
var flags = InputBuffer.ReadLeByte();
if (flags < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(flags);
/* This flag byte is divided into individual bits as follows:
*
* bit 0 FTEXT
* bit 1 FHCRC
* bit 2 FEXTRA
* bit 3 FNAME
* bit 4 FCOMMENT
* bit 5 reserved
* bit 6 reserved
* bit 7 reserved
*/
// 3.1 Check the reserved bits are zero
if ((flags & 0xE0) != 0)
{
throw new InvalidDataException("Reserved flag bits in GZIP header != 0");
}
// 4.-6. Skip the modification time, extra flags, and OS type
for (var i = 0; i < 6; i++)
{
var readByte = InputBuffer.ReadLeByte();
if (readByte < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(readByte);
}
// 7. Read extra field
if ((flags & (int)GZipFlags.Extra) != 0)
{
// XLEN is total length of extra subfields, we will skip them all
int len1, len2;
len1 = InputBuffer.ReadLeByte();
len2 = InputBuffer.ReadLeByte();
if ((len1 < 0) || (len2 < 0))
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(len1);
headCRC.Update(len2);
var extraLen = (len2 << 8) | len1; // gzip is LSB first
for (var i = 0; i < extraLen; i++)
{
var readByte = InputBuffer.ReadLeByte();
if (readByte < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(readByte);
}
}
// 8. Read file name
if ((flags & (int)GZipFlags.Name) != 0)
{
int readByte;
while ((readByte = InputBuffer.ReadLeByte()) > 0)
{
headCRC.Update(readByte);
}
if (readByte < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(readByte);
}
// 9. Read comment
if ((flags & (int)GZipFlags.Comment) != 0)
{
int readByte;
while ((readByte = InputBuffer.ReadLeByte()) > 0)
{
headCRC.Update(readByte);
}
if (readByte < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
headCRC.Update(readByte);
}
// 10. Read header CRC
if ((flags & (int)GZipFlags.CRC) != 0)
{
int tempByte;
var crcval = InputBuffer.ReadLeByte();
if (crcval < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
tempByte = InputBuffer.ReadLeByte();
if (tempByte < 0)
{
throw new EndOfStreamException("EOS reading GZIP header");
}
crcval = (crcval << 8) | tempByte;
if (crcval != ((int)headCRC.Value & 0xffff))
{
throw new InvalidDataException("Header CRC value mismatch");
}
}
readGZIPHeader = true;
return(true);
}
public static async Task <long> CopyBetweenFileBaseAsync(FileBase src, FileBase dest, CopyFileParams?param = null, ProgressReporterBase?reporter = null,
long estimatedSize = -1, CancellationToken cancel = default, RefBool?readErrorIgnored = null, Ref <uint>?srcZipCrc = null, long truncateSize = -1)
{
if (param == null)
{
param = new CopyFileParams();
}
if (reporter == null)
{
reporter = new NullProgressReporter(null);
}
if (readErrorIgnored == null)
{
readErrorIgnored = new RefBool();
}
if (srcZipCrc == null)
{
srcZipCrc = new Ref <uint>();
}
if (estimatedSize < 0)
{
estimatedSize = src.Size;
}
if (truncateSize >= 0)
{
estimatedSize = Math.Min(estimatedSize, truncateSize);
}
ZipCrc32 srcCrc = new ZipCrc32();
readErrorIgnored.Set(false);
checked
{
long currentPosition = 0;
if (param.IgnoreReadError)
{
long fileSize = src.Size;
int errorCounter = 0;
if (truncateSize >= 0)
{
fileSize = truncateSize; // Truncate
}
// Ignore read error mode
using (MemoryHelper.FastAllocMemoryWithUsing(param.IgnoreReadErrorSectorSize, out Memory <byte> buffer))
{
for (long pos = 0; pos < fileSize; pos += param.IgnoreReadErrorSectorSize)
{
int size = (int)Math.Min(param.IgnoreReadErrorSectorSize, (fileSize - pos));
Memory <byte> buffer2 = buffer.Slice(0, size);
int readSize = 0;
try
{
//if (pos >= 10000000 && pos <= (10000000 + 100000)) throw new ApplicationException("*err*");
readSize = await src.ReadRandomAsync(pos, buffer2, cancel);
}
catch (Exception ex)
{
errorCounter++;
if (errorCounter >= 100)
{
// Skip error display
if (errorCounter == 100)
{
Con.WriteError($"The read error counter of the file \"{src.FileParams.Path}\" exceeds 100. No more errors will be reported.");
}
}
else
{
// Display the error
Con.WriteError($"Ignoring the read error at the offset {pos} of the file \"{src.FileParams.Path}\". Error: {ex.Message}");
}
readErrorIgnored.Set(true);
}
if (readSize >= 1)
{
await dest.WriteRandomAsync(pos, buffer2.Slice(0, readSize), cancel);
}
currentPosition = pos + readSize;
reporter.ReportProgress(new ProgressData(currentPosition, fileSize));
}
}
}
else if (param.AsyncCopy == false)
{
// Normal copy
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer))
{
while (true)
{
Memory <byte> thisTimeBuffer = buffer;
if (truncateSize >= 0)
{
// Truncate
long remainSize = Math.Max(truncateSize - currentPosition, 0);
if (thisTimeBuffer.Length > remainSize)
{
thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
}
if (remainSize == 0)
{
break;
}
}
int readSize = await src.ReadAsync(thisTimeBuffer, cancel);
Debug.Assert(readSize <= thisTimeBuffer.Length);
if (readSize <= 0)
{
break;
}
ReadOnlyMemory <byte> sliced = thisTimeBuffer.Slice(0, readSize);
if (param.Flags.Bit(FileFlags.CopyFile_Verify))
{
srcCrc.Append(sliced.Span);
}
await dest.WriteAsync(sliced, cancel);
currentPosition += readSize;
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
}
}
else
{
// Async copy
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer1))
{
using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer2))
{
Task?lastWriteTask = null;
int number = 0;
int writeSize = 0;
long currentReadPosition = 0;
Memory <byte>[] buffers = new Memory <byte>[2] {
buffer1, buffer2
};
while (true)
{
Memory <byte> buffer = buffers[(number++) % 2];
Memory <byte> thisTimeBuffer = buffer;
if (truncateSize >= 0)
{
// Truncate
long remainSize = Math.Max(truncateSize - currentReadPosition, 0);
if (thisTimeBuffer.Length > remainSize)
{
thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize);
}
}
int readSize = await src.ReadAsync(thisTimeBuffer, cancel);
Debug.Assert(readSize <= buffer.Length);
if (lastWriteTask != null)
{
await lastWriteTask;
currentPosition += writeSize;
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
if (readSize <= 0)
{
break;
}
currentReadPosition += readSize;
writeSize = readSize;
ReadOnlyMemory <byte> sliced = buffer.Slice(0, writeSize);
if (param.Flags.Bit(FileFlags.CopyFile_Verify))
{
srcCrc.Append(sliced.Span);
}
lastWriteTask = dest.WriteAsync(sliced, cancel);
}
reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize));
}
}
}
srcZipCrc.Set(srcCrc.Value);
return(currentPosition);
}
}
