private void CompressionWorker(BlockingCollection <Block> toCompress, BlockingCollection <Block> toWrite, int columns, OrderedWaiter waiter,
ExceptionMarshaller exMarshaller)
{
Contracts.AssertValue(exMarshaller);
try
{
_host.AssertValue(toCompress);
_host.AssertValue(toWrite);
_host.Assert(columns > 0);
_host.Assert(_deterministicBlockOrder == (waiter != null));
foreach (Block block in toCompress.GetConsumingEnumerable(exMarshaller.Token))
{
MemoryStream compressed = _memPool.Get();
int uncompLength;
using (Stream stream = _compression.CompressStream(compressed))
{
MemoryStream uncompressed = block.BlockData;
uncompLength = (int)uncompressed.Length;
ArraySegment <byte> buffer;
bool tmp = uncompressed.TryGetBuffer(out buffer);
Contracts.Assert(tmp);
stream.Write(buffer.Array, buffer.Offset, buffer.Count);
_memPool.Return(ref uncompressed);
}
if (_deterministicBlockOrder)
{
waiter.Wait((long)columns * block.BlockIndex + block.ColumnIndex, exMarshaller.Token);
}
toWrite.Add(new Block(compressed, block.ColumnIndex, block.BlockIndex, uncompLength), exMarshaller.Token);
if (_deterministicBlockOrder)
{
waiter.Increment();
}
}
}
catch (Exception ex)
{
exMarshaller.Set("compressing", ex);
}
}
public IEnumerable <RowBatch> GetBatches()
{
_waiterReading.Increment();
_waiterPublish.Increment();
return(_queue.GetConsumingEnumerable());
}
