Beispiel #1
0
        public static void AddColumn(this StripeFooter footer, ColumnEncodingKind columnEncodingKind,
                                     uint dictionarySize = 0)
        {
            var columnEncoding = new ColumnEncoding
            {
                Kind           = columnEncodingKind,
                DictionarySize = dictionarySize
            };

            footer.Columns.Add(columnEncoding);
        }
Beispiel #2
0
 internal StripeStreamReader(Stream inputStream, uint columnId, StreamKind streamKind,
                             ColumnEncodingKind encodingKind, long inputStreamOffset, ulong compressedLength,
                             CompressionKind compressionKind)
 {
     _inputStream       = inputStream;
     ColumnId           = columnId;
     StreamKind         = streamKind;
     ColumnEncodingKind = encodingKind;
     _inputStreamOffset = inputStreamOffset;
     _compressedLength  = compressedLength;
     _compressionKind   = compressionKind;
 }
Beispiel #3
0
        private void EnsureEncodingKindIsSet(IList <string> values)
        {
            if (ColumnEncodingIsValid())
            {
                return;
            }

            //Detect the encoding type
            var nonNullValues = values.Where(v => v != null);
            var uniqueValues  = nonNullValues.Distinct().Count();
            var totalValues   = nonNullValues.Count();

            ColumnEncoding = (double)uniqueValues / (double)totalValues <= _uniqueStringThresholdRatio
                ? ColumnEncodingKind.DictionaryV2
                : ColumnEncodingKind.DirectV2;
        }