Esempio n. 1
0
        void CompleteStripe()
        {
            var stripeFooter = new Protocol.StripeFooter();
            var stripeStats  = new Protocol.StripeStatistics();

            //Columns
            foreach (var writer in _columnWriters)
            {
                writer.ColumnWriter.FlushBuffers();
                var dictionaryLength = (writer.ColumnWriter as ColumnTypes.StringWriter)?.DictionaryLength ?? 0;                        //DictionaryLength is only used by StringWriter
                stripeFooter.AddColumn(writer.ColumnWriter.ColumnEncoding, dictionaryLength);
            }

            var stripeInformation = new Protocol.StripeInformation();

            stripeInformation.Offset       = (ulong)_outputStream.Position;
            stripeInformation.NumberOfRows = (ulong)_rowsInStripe;

            //Indexes
            foreach (var writer in _columnWriters)
            {
                //Write the index buffer
                var indexBuffer = _bufferFactory.CreateBuffer(Protocol.StreamKind.RowIndex);
                writer.ColumnWriter.Statistics.WriteToBuffer(indexBuffer, i => writer.ColumnWriter.Buffers[i].MustBeIncluded);
                indexBuffer.CopyTo(_outputStream);

                //Add the index to the footer
                stripeFooter.AddDataStream(writer.ColumnWriter.ColumnId, indexBuffer);

                //Collect summary statistics
                var columnStats = new ColumnStatistics();
                foreach (var stats in writer.ColumnWriter.Statistics)
                {
                    stats.FillColumnStatistics(columnStats);
                    stats.FillColumnStatistics(writer.FileStatistics);
                }
                stripeStats.ColStats.Add(columnStats);
            }
            _stripeStats.Add(stripeStats);

            stripeInformation.IndexLength = (ulong)_outputStream.Position - stripeInformation.Offset;

            //Data streams
            foreach (var writer in _columnWriters)
            {
                foreach (var buffer in writer.ColumnWriter.Buffers)
                {
                    if (!buffer.MustBeIncluded)
                    {
                        continue;
                    }
                    buffer.CopyTo(_outputStream);
                    stripeFooter.AddDataStream(writer.ColumnWriter.ColumnId, buffer);
                }
            }

            stripeInformation.DataLength = (ulong)_outputStream.Position - stripeInformation.IndexLength - stripeInformation.Offset;

            //Footer
            long footerLength;

            _bufferFactory.SerializeAndCompressTo(_outputStream, stripeFooter, out footerLength);
            stripeInformation.FooterLength = (ulong)footerLength;

            _stripeInformations.Add(stripeInformation);

            _rowsInFile  += _rowsInStripe;
            _rowsInStripe = 0;
            foreach (var writer in _columnWriters)
            {
                writer.ColumnWriter.Reset();
            }
        }