private void CompressionWorker(BlockingCollection <Block> toCompress, BlockingCollection <Block> toWrite, int columns, OrderedWaiter waiter, ExceptionMarshaller exMarshaller) { Contracts.AssertValue(exMarshaller); try { _host.AssertValue(toCompress); _host.AssertValue(toWrite); _host.Assert(columns > 0); _host.Assert(_deterministicBlockOrder == (waiter != null)); foreach (Block block in toCompress.GetConsumingEnumerable(exMarshaller.Token)) { MemoryStream compressed = _memPool.Get(); int uncompLength; using (Stream stream = _compression.CompressStream(compressed)) { MemoryStream uncompressed = block.BlockData; uncompLength = (int)uncompressed.Length; ArraySegment <byte> buffer; bool tmp = uncompressed.TryGetBuffer(out buffer); Contracts.Assert(tmp); stream.Write(buffer.Array, buffer.Offset, buffer.Count); _memPool.Return(ref uncompressed); } if (_deterministicBlockOrder) { waiter.Wait((long)columns * block.BlockIndex + block.ColumnIndex, exMarshaller.Token); } toWrite.Add(new Block(compressed, block.ColumnIndex, block.BlockIndex, uncompLength), exMarshaller.Token); if (_deterministicBlockOrder) { waiter.Increment(); } } } catch (Exception ex) { exMarshaller.Set("compressing", ex); } }
public IEnumerable <RowBatch> GetBatches() { _waiterReading.Increment(); _waiterPublish.Increment(); return(_queue.GetConsumingEnumerable()); }