private void CreateLists(SchemaElement schema, IList[] hl, int rl)
{
int maxIdx = schema.MaxRepetitionLevel - 1;
//replace lists in chain with new instances
for (int i = maxIdx; i >= rl; i--)
{
IList nl = (i == maxIdx)
? schema.CreateValuesList(0)
: new List <IList>();
hl[i] = nl;
}
//rightest old list now should point to leftest new list
if (rl > 0 && rl <= maxIdx)
{
hl[rl - 1].Add(hl[rl]);
}
//chain new lists together
for (int i = maxIdx - 1; i >= rl; i--)
{
hl[i].Add(hl[i + 1]);
}
}
public IList Unpack(SchemaElement schema, IList hierarchy, out List <int> levels)
{
levels = new List <int>();
IList flatValues = schema.CreateValuesList(0);
int touched = 0;
Unpack(schema.MaxRepetitionLevel, hierarchy, levels, flatValues, ref touched, 0);
return(flatValues);
}
public IList Read(long offset, long count)
{
//get the minimum offset, we'll just read pages in sequence
long fileOffset = new[] { _thriftChunk.Meta_data.Dictionary_page_offset, _thriftChunk.Meta_data.Data_page_offset }.Where(e => e != 0).Min();
long maxValues = _thriftChunk.Meta_data.Num_values;
//IList values = TypeFactory.Create(_schema, false, (int)maxValues);
IList values = _schema.CreateValuesList((int)maxValues);
_inputStream.Seek(fileOffset, SeekOrigin.Begin);
Thrift.PageHeader ph = _thrift.Read <Thrift.PageHeader>();
IList dictionaryPage = null;
List <int> indexes = null;
List <int> definitions = null;
List <int> repetitions = null;
//there can be only one dictionary page in column
if (ph.Type == Thrift.PageType.DICTIONARY_PAGE)
{
dictionaryPage = ReadDictionaryPage(ph);
ph = _thrift.Read <Thrift.PageHeader>(); //get next page after dictionary
}
int dataPageCount = 0;
while (true)
{
int valuesSoFar = Math.Max(indexes == null ? 0 : indexes.Count, values.Count);
PageData page = ReadDataPage(ph, values, maxValues - valuesSoFar);
indexes = AssignOrAdd(indexes, page.indexes);
definitions = AssignOrAdd(definitions, page.definitions);
repetitions = AssignOrAdd(repetitions, page.repetitions);
dataPageCount++;
int totalCount = Math.Max(
values.Count +
(indexes == null
? 0
: indexes.Count),
definitions == null ? 0 : definitions.Count);
if (totalCount >= maxValues)
{
break; //limit reached
}
ph = ReadDataPageHeader(dataPageCount); //get next page
if (ph.Type != Thrift.PageType.DATA_PAGE)
{
break;
}
}
IList mergedValues = new ValueMerger(_schema, values)
.Apply(dictionaryPage, definitions, repetitions, indexes, (int)maxValues);
//todo: this won't work for nested arrays
ValueMerger.Trim(mergedValues, (int)offset, (int)count);
return(mergedValues);
}