public Block Compress(Block inputBlock, DecompressionHelpMode decompressionHelpMode)
{
if (inputBlock.Bytes.Count == 0)
{
return(new Block(EmptyBytes, inputBlock.Offset));
}
using (var memoryStream = new MemoryStream())
{
using (var gZipStream = new GZipStream(memoryStream, CompressionMode.Compress, true))
{
// ReSharper disable once AssignNullToNotNullAttribute
gZipStream.Write(inputBlock.Bytes.Array, inputBlock.Bytes.Offset, inputBlock.Bytes.Count);
}
var compressedBytes = memoryStream.ToArray();
var buffer = new ArraySegment <byte>(compressedBytes, 0, (int)memoryStream.Position);
if (decompressionHelpMode == DecompressionHelpMode.BlockLengthInMimetypeSection)
{
GzipHeader.FindFirst(buffer)?.SetMimetypeBytes(BitConverter.GetBytes(buffer.Count));
}
return(new Block(buffer, inputBlock.Offset));
}
}
public void Should_return_null_when_bytes_does_not_contain_gzip_header()
{
var bytes = TestData.ShortFileContent.ToArraySegment();
var result = GzipHeader.FindFirst(bytes);
result.Should().BeNull();
}
public void Should_return_gzip_header_when_bytes_have_size_exactly_equal_to_header()
{
var bytes = TestData.ShortFileContent.GzipCompress().TakeFirst(GzipHeader.Length);
var result = GzipHeader.FindFirst(bytes);
result.Should().NotBeNull();
result.Value.Position.Should().Be(0);
}
public void Should_return_gzip_header_when_bytes_starts_with_gzip_header()
{
var bytes = TestData.ShortFileContent.GzipCompress();
var result = GzipHeader.FindFirst(bytes);
result.Should().NotBeNull();
result.Value.Position.Should().Be(0);
}
public void Should_set_length_into_mime_type_section_when_enabled_set_length_to_mimetype()
{
var originalString = TestData.InputFileContent;
var originalBytes = originalString.ToArraySegment();
var block = new Block(originalBytes, 100);
var compressedBlock = gzipBlockCompression.Compress(block, DecompressionHelpMode.BlockLengthInMimetypeSection);
var mimetype = GzipHeader.FindFirst(compressedBlock.Bytes).Value.MimetypeBytes.ToArray();
BitConverter.ToInt32(mimetype).Should().Be(compressedBlock.Bytes.Count);
}
public void Should_return_null_when_given_segment_have_less_count_than_header_required()
{
var someUncompressedBytes = TestData.ShortFileContent.ToBytes();
var firstCompressed = TestData.ShortFileContent.GzipCompress();
var bytes = someUncompressedBytes.Concat(firstCompressed)
.ToArray()
.Segment(someUncompressedBytes.Length, 8);
var result = GzipHeader.FindFirst(bytes);
result.Should().BeNull();
}
public void Should_return_gzip_header_when_it_first_occurence_not_from_start_of_bytes()
{
var someUncompressedBytes = TestData.ShortFileContent.ToBytes();
var bytes = someUncompressedBytes.Concat(TestData.ShortFileContent.GzipCompress())
.Concat(TestData.ShortFileContent.GzipCompress())
.ToArray()
.ToSegment();
var result = GzipHeader.FindFirst(bytes);
result.Should().NotBeNull();
result.Value.Position.Should().Be(someUncompressedBytes.Length);
}
public void Should_set_position_of_found_header_with_respecting_segment_offset()
{
var uncompressedBytes = TestData.ShortFileContent.ToBytes();
var firstCompressed = TestData.ShortFileContent.GzipCompress();
var secondCompressed = TestData.ShortFileContent.GzipCompress();
var bytes = uncompressedBytes
.Concat(firstCompressed)
.Concat(secondCompressed)
.ToArray()
.Segment(uncompressedBytes.Length + 3);
var result = GzipHeader.FindFirst(bytes);
result.Should().NotBeNull();
result.Value.Position.Should().Be(uncompressedBytes.Length + firstCompressed.Count);
}
public IEnumerable <SplitResult> SplitToIndependentBlocks(RewindableReadonlyStream inputStream)
{
long GetCurrentHeaderPosition(ArraySegment <byte> headerBytes)
{
return(inputStream.Position - headerBytes.Offset);
}
Result <int, GzipSplittingStatus?> GetBlockLengthFromGzipHeader(ArraySegment <byte> headerBytes)
{
var header = GzipHeader.FindFirst(headerBytes);
if (header == null)
{
return(GzipSplittingStatus.WrongFormat);
}
var length = header.Value.GetMimetypeAsInt();
if (length == 0)
{
return(GzipSplittingStatus.CantReadBlock);
}
if (length > blockSizeLimit)
{
log.Info($"Block size {length} at offset {GetCurrentHeaderPosition(headerBytes)} is greater than limit {blockSizeLimit}");
return(GzipSplittingStatus.CantReadBlock);
}
return(length);
}
(int length, ArraySegment <byte> readBytes, bool endOfStream, GzipSplittingStatus?error) ReadCurrentGzipBlockLengthFromStream()
{
var length = 0;
GzipSplittingStatus?readLengthError = null;
var bytes = new byte[GzipHeader.Length];
var(headerBytes, isEndOfStream) = inputStream.ReadExactBuffer(bytes);
if (headerBytes.Count < bytes.Length)
{
if (!isEndOfStream)
{
readLengthError = GzipSplittingStatus.WrongFormat;
}
}
else
{
(length, readLengthError) = GetBlockLengthFromGzipHeader(headerBytes);
}
return(length, headerBytes, isEndOfStream, readLengthError);
}
var currentBlockPoistion = inputStream.Position;
var(blockLength, headerReadBytes, endOfStream, error) = ReadCurrentGzipBlockLengthFromStream();
if (headerReadBytes.Count > 0)
{
inputStream.ReturnTailOfReadedBytes(headerReadBytes);
}
if (error.HasValue)
{
yield return(error.Value);
yield break;
}
while (!endOfStream)
{
ArraySegment <byte> buffer;
var blockAndNextHeaderLength = blockLength + GzipHeader.Length;
(buffer, endOfStream) = inputStream.ReadExactFullBuffer(blockAndNextHeaderLength);
if (!endOfStream && buffer.Count == 0)
{
log.Info($"Block hader at offset {currentBlockPoistion} contain invalid length or non length in mime type");
yield return(GzipSplittingStatus.CantReadBlock);
yield break;
}
currentBlockPoistion = inputStream.Position;
ArraySegment <byte> independentBlockBytes;
var nextBlockLength = 0;
GzipSplittingStatus?nextHeaderError = null;
if (buffer.Count == blockLength && endOfStream)
{
independentBlockBytes = buffer;
}
else if (buffer.Count == blockAndNextHeaderLength)
{
var headerBytes = buffer.SliceFromEnd(GzipHeader.Length);
(nextBlockLength, nextHeaderError) = GetBlockLengthFromGzipHeader(headerBytes);
if (nextHeaderError != GzipSplittingStatus.WrongFormat)
{
inputStream.ReturnTailOfReadedBytes(headerBytes);
independentBlockBytes = buffer.RemoveFromEnd(GzipHeader.Length);
}
}
if (independentBlockBytes.Count == 0)
{
inputStream.ReturnTailOfReadedBytes(buffer);
yield return(GzipSplittingStatus.CantReadBlock);
yield break;
}
yield return(new IndependentGzipBlock(WrapBufferWithStream(independentBlockBytes)));
if (nextHeaderError.HasValue)
{
yield return(nextHeaderError.Value);
yield break;
}
blockLength = nextBlockLength;
}
yield return(GzipSplittingStatus.StreamIsEnd);
}
private static void SetLengthToHeader(byte[] bytes, int length)
{
// ReSharper disable once PossibleInvalidOperationException
GzipHeader.FindFirst(bytes).Value.SetMimetypeBytes(BitConverter.GetBytes(length));
}