static async Task <uint> CalcZipCrc32HandleAsync(FileBase src, CopyFileParams param, CancellationToken cancel) { ZipCrc32 srcCrc = new ZipCrc32(); using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer)) { while (true) { int readSize = await src.ReadAsync(buffer, cancel); Debug.Assert(readSize <= buffer.Length); if (readSize <= 0) { break; } ReadOnlyMemory <byte> sliced = buffer.Slice(0, readSize); srcCrc.Append(sliced.Span); } } return(srcCrc.Value); }
void UpdateKeys(byte c) { Key0 = ZipCrc32.CalcCrc32ForZipEncryption(Key0, c); Key1 += (byte)Key0; Key1 = Key1 * 134775813 + 1; Key2 = ZipCrc32.CalcCrc32ForZipEncryption(Key2, (byte)(Key1 >> 24)); }
public void ZipCrc32() { var underTestCrc32 = new ZipCrc32(); Assert.AreEqual(0x0, underTestCrc32.Value); underTestCrc32.Update(check); Assert.AreEqual(0xCBF43926, underTestCrc32.Value); underTestCrc32.Reset(); Assert.AreEqual(0x0, underTestCrc32.Value); exceptionTesting(underTestCrc32); }
bool ReadHeader() { // Initialize CRC for this block crc = new ZipCrc32(); // Make sure there is data in file. We can't rely on ReadLeByte() to fill the buffer, as this could be EOF, // which is fine, but ReadLeByte() throws an exception if it doesn't find data, so we do this part ourselves. if (InputBuffer.Available <= 0) { InputBuffer.Fill(); if (InputBuffer.Available <= 0) { // No header, EOF. return(false); } } // 1. Check the two magic bytes var headCRC = new ZipCrc32(); var magic = InputBuffer.ReadLeByte(); if (magic < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(magic); if (magic != (GZipConstants.MAGIC >> 8)) { throw new InvalidDataException("Error GZIP header, first magic byte doesn't match"); } // magic = baseInputStream.ReadByte(); magic = InputBuffer.ReadLeByte(); if (magic < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } if (magic != (GZipConstants.MAGIC & 0xFF)) { throw new InvalidDataException("Error GZIP header, second magic byte doesn't match"); } headCRC.Update(magic); // 2. Check the compression type (must be 8) var compressionType = InputBuffer.ReadLeByte(); if (compressionType < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } if (compressionType != 8) { throw new InvalidDataException("Error GZIP header, data not in deflate format"); } headCRC.Update(compressionType); // 3. Check the flags var flags = InputBuffer.ReadLeByte(); if (flags < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(flags); /* This flag byte is divided into individual bits as follows: * * bit 0 FTEXT * bit 1 FHCRC * bit 2 FEXTRA * bit 3 FNAME * bit 4 FCOMMENT * bit 5 reserved * bit 6 reserved * bit 7 reserved */ // 3.1 Check the reserved bits are zero if ((flags & 0xE0) != 0) { throw new InvalidDataException("Reserved flag bits in GZIP header != 0"); } // 4.-6. Skip the modification time, extra flags, and OS type for (var i = 0; i < 6; i++) { var readByte = InputBuffer.ReadLeByte(); if (readByte < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(readByte); } // 7. Read extra field if ((flags & (int)GZipFlags.Extra) != 0) { // XLEN is total length of extra subfields, we will skip them all int len1, len2; len1 = InputBuffer.ReadLeByte(); len2 = InputBuffer.ReadLeByte(); if ((len1 < 0) || (len2 < 0)) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(len1); headCRC.Update(len2); var extraLen = (len2 << 8) | len1; // gzip is LSB first for (var i = 0; i < extraLen; i++) { var readByte = InputBuffer.ReadLeByte(); if (readByte < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(readByte); } } // 8. Read file name if ((flags & (int)GZipFlags.Name) != 0) { int readByte; while ((readByte = InputBuffer.ReadLeByte()) > 0) { headCRC.Update(readByte); } if (readByte < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(readByte); } // 9. Read comment if ((flags & (int)GZipFlags.Comment) != 0) { int readByte; while ((readByte = InputBuffer.ReadLeByte()) > 0) { headCRC.Update(readByte); } if (readByte < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } headCRC.Update(readByte); } // 10. Read header CRC if ((flags & (int)GZipFlags.CRC) != 0) { int tempByte; var crcval = InputBuffer.ReadLeByte(); if (crcval < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } tempByte = InputBuffer.ReadLeByte(); if (tempByte < 0) { throw new EndOfStreamException("EOS reading GZIP header"); } crcval = (crcval << 8) | tempByte; if (crcval != ((int)headCRC.Value & 0xffff)) { throw new InvalidDataException("Header CRC value mismatch"); } } readGZIPHeader = true; return(true); }
public static async Task <long> CopyBetweenFileBaseAsync(FileBase src, FileBase dest, CopyFileParams?param = null, ProgressReporterBase?reporter = null, long estimatedSize = -1, CancellationToken cancel = default, RefBool?readErrorIgnored = null, Ref <uint>?srcZipCrc = null, long truncateSize = -1) { if (param == null) { param = new CopyFileParams(); } if (reporter == null) { reporter = new NullProgressReporter(null); } if (readErrorIgnored == null) { readErrorIgnored = new RefBool(); } if (srcZipCrc == null) { srcZipCrc = new Ref <uint>(); } if (estimatedSize < 0) { estimatedSize = src.Size; } if (truncateSize >= 0) { estimatedSize = Math.Min(estimatedSize, truncateSize); } ZipCrc32 srcCrc = new ZipCrc32(); readErrorIgnored.Set(false); checked { long currentPosition = 0; if (param.IgnoreReadError) { long fileSize = src.Size; int errorCounter = 0; if (truncateSize >= 0) { fileSize = truncateSize; // Truncate } // Ignore read error mode using (MemoryHelper.FastAllocMemoryWithUsing(param.IgnoreReadErrorSectorSize, out Memory <byte> buffer)) { for (long pos = 0; pos < fileSize; pos += param.IgnoreReadErrorSectorSize) { int size = (int)Math.Min(param.IgnoreReadErrorSectorSize, (fileSize - pos)); Memory <byte> buffer2 = buffer.Slice(0, size); int readSize = 0; try { //if (pos >= 10000000 && pos <= (10000000 + 100000)) throw new ApplicationException("*err*"); readSize = await src.ReadRandomAsync(pos, buffer2, cancel); } catch (Exception ex) { errorCounter++; if (errorCounter >= 100) { // Skip error display if (errorCounter == 100) { Con.WriteError($"The read error counter of the file \"{src.FileParams.Path}\" exceeds 100. No more errors will be reported."); } } else { // Display the error Con.WriteError($"Ignoring the read error at the offset {pos} of the file \"{src.FileParams.Path}\". Error: {ex.Message}"); } readErrorIgnored.Set(true); } if (readSize >= 1) { await dest.WriteRandomAsync(pos, buffer2.Slice(0, readSize), cancel); } currentPosition = pos + readSize; reporter.ReportProgress(new ProgressData(currentPosition, fileSize)); } } } else if (param.AsyncCopy == false) { // Normal copy using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer)) { while (true) { Memory <byte> thisTimeBuffer = buffer; if (truncateSize >= 0) { // Truncate long remainSize = Math.Max(truncateSize - currentPosition, 0); if (thisTimeBuffer.Length > remainSize) { thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize); } if (remainSize == 0) { break; } } int readSize = await src.ReadAsync(thisTimeBuffer, cancel); Debug.Assert(readSize <= thisTimeBuffer.Length); if (readSize <= 0) { break; } ReadOnlyMemory <byte> sliced = thisTimeBuffer.Slice(0, readSize); if (param.Flags.Bit(FileFlags.CopyFile_Verify)) { srcCrc.Append(sliced.Span); } await dest.WriteAsync(sliced, cancel); currentPosition += readSize; reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize)); } } } else { // Async copy using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer1)) { using (MemoryHelper.FastAllocMemoryWithUsing(param.BufferSize, out Memory <byte> buffer2)) { Task?lastWriteTask = null; int number = 0; int writeSize = 0; long currentReadPosition = 0; Memory <byte>[] buffers = new Memory <byte>[2] { buffer1, buffer2 }; while (true) { Memory <byte> buffer = buffers[(number++) % 2]; Memory <byte> thisTimeBuffer = buffer; if (truncateSize >= 0) { // Truncate long remainSize = Math.Max(truncateSize - currentReadPosition, 0); if (thisTimeBuffer.Length > remainSize) { thisTimeBuffer = thisTimeBuffer.Slice(0, (int)remainSize); } } int readSize = await src.ReadAsync(thisTimeBuffer, cancel); Debug.Assert(readSize <= buffer.Length); if (lastWriteTask != null) { await lastWriteTask; currentPosition += writeSize; reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize)); } if (readSize <= 0) { break; } currentReadPosition += readSize; writeSize = readSize; ReadOnlyMemory <byte> sliced = buffer.Slice(0, writeSize); if (param.Flags.Bit(FileFlags.CopyFile_Verify)) { srcCrc.Append(sliced.Span); } lastWriteTask = dest.WriteAsync(sliced, cancel); } reporter.ReportProgress(new ProgressData(currentPosition, estimatedSize)); } } } srcZipCrc.Set(srcCrc.Value); return(currentPosition); } }