Exemplo n.º 1
0
        private void CompressionWorker(BlockingCollection <Block> toCompress, BlockingCollection <Block> toWrite, int columns, OrderedWaiter waiter,
                                       ExceptionMarshaller exMarshaller)
        {
            Contracts.AssertValue(exMarshaller);
            try
            {
                _host.AssertValue(toCompress);
                _host.AssertValue(toWrite);
                _host.Assert(columns > 0);
                _host.Assert(_deterministicBlockOrder == (waiter != null));

                foreach (Block block in toCompress.GetConsumingEnumerable(exMarshaller.Token))
                {
                    MemoryStream compressed = _memPool.Get();
                    int          uncompLength;
                    using (Stream stream = _compression.CompressStream(compressed))
                    {
                        MemoryStream uncompressed = block.BlockData;
                        uncompLength = (int)uncompressed.Length;
                        ArraySegment <byte> buffer;
                        bool tmp = uncompressed.TryGetBuffer(out buffer);
                        Contracts.Assert(tmp);
                        stream.Write(buffer.Array, buffer.Offset, buffer.Count);
                        _memPool.Return(ref uncompressed);
                    }
                    if (_deterministicBlockOrder)
                    {
                        waiter.Wait((long)columns * block.BlockIndex + block.ColumnIndex, exMarshaller.Token);
                    }
                    toWrite.Add(new Block(compressed, block.ColumnIndex, block.BlockIndex, uncompLength), exMarshaller.Token);
                    if (_deterministicBlockOrder)
                    {
                        waiter.Increment();
                    }
                }
            }
            catch (Exception ex)
            {
                exMarshaller.Set("compressing", ex);
            }
        }
Exemplo n.º 2
0
                private void Parse(int tid)
                {
                    long blk  = tid;
                    int  iblk = tid;

                    Contracts.Assert(iblk < _blockCount - 3);

                    var helper = _curs._parser.CreateHelper(_rows.Stats, _curs._srcNeeded);

                    while (!_done)
                    {
                        // Algorithm:
                        // * When it is our turn, grab a block of lines.
                        // * Parse rows.
                        // * When it is our turn, enqueue the batch.

                        // When it is our turn, read the lines and signal the next worker that it is ok to read.
                        LineBatch lines;
                        _waiterReading.Wait(blk);
                        if (_done)
                        {
                            return;
                        }
                        try
                        {
                            lines = _reader.GetBatch();
                        }
                        finally
                        {
                            _waiterReading.Increment();
                        }
                        Contracts.Assert(lines.Exception == null);
                        if (lines.Infos == null || _done)
                        {
                            return;
                        }

                        // Parse the lines into rows.
                        Contracts.Assert(lines.Infos.Length <= BlockSize);

                        var batch = new RowBatch(iblk * BlockSize, iblk * BlockSize + lines.Infos.Length, lines.Total);
                        int irow  = batch.IrowMin;
                        foreach (var info in lines.Infos)
                        {
                            Contracts.Assert(info.Line > 0);
                            Contracts.AssertNonEmpty(info.Text);
                            if (_done)
                            {
                                return;
                            }
                            _curs._parser.ParseRow(_rows, irow, helper, _curs._active, lines.Path, info.Line, info.Text);
                            irow++;
                        }
                        Contracts.Assert(irow == batch.IrowLim);

                        if (_done)
                        {
                            return;
                        }

                        // When it is our turn, publish the rows and signal the next worker that it is ok to publish.
                        _waiterPublish.Wait(blk);
                        if (_done)
                        {
                            return;
                        }
                        while (!_queue.TryAdd(batch, TimeOut))
                        {
                            if (_done)
                            {
                                return;
                            }
                        }
                        _waiterPublish.Increment();

                        blk  += _threads.Length;
                        iblk += _threads.Length;
                        if (iblk >= _blockCount)
                        {
                            iblk -= _blockCount;
                        }
                        Contracts.Assert(0 <= iblk && iblk < _blockCount);
                    }
                }