private void CreateLists(SchemaElement schema, IList[] hl, int rl) { int maxIdx = schema.MaxRepetitionLevel - 1; //replace lists in chain with new instances for (int i = maxIdx; i >= rl; i--) { IList nl = (i == maxIdx) ? schema.CreateValuesList(0) : new List <IList>(); hl[i] = nl; } //rightest old list now should point to leftest new list if (rl > 0 && rl <= maxIdx) { hl[rl - 1].Add(hl[rl]); } //chain new lists together for (int i = maxIdx - 1; i >= rl; i--) { hl[i].Add(hl[i + 1]); } }
public IList Unpack(SchemaElement schema, IList hierarchy, out List <int> levels) { levels = new List <int>(); IList flatValues = schema.CreateValuesList(0); int touched = 0; Unpack(schema.MaxRepetitionLevel, hierarchy, levels, flatValues, ref touched, 0); return(flatValues); }
public IList Read(long offset, long count) { //get the minimum offset, we'll just read pages in sequence long fileOffset = new[] { _thriftChunk.Meta_data.Dictionary_page_offset, _thriftChunk.Meta_data.Data_page_offset }.Where(e => e != 0).Min(); long maxValues = _thriftChunk.Meta_data.Num_values; //IList values = TypeFactory.Create(_schema, false, (int)maxValues); IList values = _schema.CreateValuesList((int)maxValues); _inputStream.Seek(fileOffset, SeekOrigin.Begin); Thrift.PageHeader ph = _thrift.Read <Thrift.PageHeader>(); IList dictionaryPage = null; List <int> indexes = null; List <int> definitions = null; List <int> repetitions = null; //there can be only one dictionary page in column if (ph.Type == Thrift.PageType.DICTIONARY_PAGE) { dictionaryPage = ReadDictionaryPage(ph); ph = _thrift.Read <Thrift.PageHeader>(); //get next page after dictionary } int dataPageCount = 0; while (true) { int valuesSoFar = Math.Max(indexes == null ? 0 : indexes.Count, values.Count); PageData page = ReadDataPage(ph, values, maxValues - valuesSoFar); indexes = AssignOrAdd(indexes, page.indexes); definitions = AssignOrAdd(definitions, page.definitions); repetitions = AssignOrAdd(repetitions, page.repetitions); dataPageCount++; int totalCount = Math.Max( values.Count + (indexes == null ? 0 : indexes.Count), definitions == null ? 0 : definitions.Count); if (totalCount >= maxValues) { break; //limit reached } ph = ReadDataPageHeader(dataPageCount); //get next page if (ph.Type != Thrift.PageType.DATA_PAGE) { break; } } IList mergedValues = new ValueMerger(_schema, values) .Apply(dictionaryPage, definitions, repetitions, indexes, (int)maxValues); //todo: this won't work for nested arrays ValueMerger.Trim(mergedValues, (int)offset, (int)count); return(mergedValues); }