public XArray Remap(XArray source, ref int[] remapArray)
{
// See if we have the remapping cached already
ArraySelector cachedMapping;
if (_cachedRemappings.TryGetValue(source.Selector, out cachedMapping))
{
return(source.Reselect(cachedMapping));
}
// Convert the BitVector to indices if we haven't yet (deferred to first column wanting values)
if (!_indicesFound)
{
_indicesFound = true;
Allocator.AllocateToSize(ref _indices, _count);
int countFound = _vector.Page(_indices, ref _nextVectorIndex, _count);
if (countFound != _count)
{
throw new InvalidOperationException($"RowRemapper found {countFound:n0} rows when {_count:n0} expected paging in Vector with {_vector.Count:n0} total matches up to index {_nextVectorIndex:n0}.");
}
}
// Remap the outer selector
XArray remapped = source.Select(ArraySelector.Map(_indices, _count), ref remapArray);
// Cache the remapping
_cachedRemappings[source.Selector] = remapped.Selector;
return(remapped);
}
private static void RoundTrip(string columnName, int[] array, int batchSize = 128)
{
XDatabaseContext context = new XDatabaseContext();
string columnPath = Path.Combine("VariableIntegerReaderWriterTests", columnName);
string columnPrefix = Path.Combine(columnPath, "Vl");
context.StreamProvider.Delete(columnPath);
Directory.CreateDirectory(columnPath);
XArray values = XArray.All(array, array.Length);
using (IColumnWriter writer = new VariableIntegerWriter(context.StreamProvider, columnPrefix))
{
ArraySelector page = ArraySelector.All(0).NextPage(array.Length, batchSize);
while (page.Count > 0)
{
writer.Append(values.Reselect(page));
page = page.NextPage(array.Length, batchSize);
}
}
XArray returned = default(XArray);
using (IColumnReader reader = new VariableIntegerReader(context.StreamProvider, columnPrefix, CachingOption.AsConfigured))
{
returned = reader.Read(ArraySelector.All(array.Length));
}
TableTestHarness.AssertAreEqual(values, returned, array.Length);
context.StreamProvider.Delete(columnPath);
}
public XArray Remap(XArray values, ArraySelector selector)
{
// Read row indices and convert to int[]
XArray indexByteArray = _rowIndexReader.Read(selector);
XArray indexIntArray = _rowIndexToIntConverter(indexByteArray);
// Return the selected values
return(values.Reselect(ArraySelector.Map((int[])indexIntArray.Array, indexIntArray.Count)));
}
private void BuildSingleEnumColumnDictionary(CancellationToken cancellationToken)
{
XArray values = _keyColumns[0].ValuesGetter()();
Func <XArray> indicesGetter = _keyColumns[0].IndicesCurrentGetter();
// Find or construct an aggregator which can track which enum values ended up with any rows in the result
IFoundIndicesTracker tracker = (IFoundIndicesTracker)_aggregators.FirstOrDefault((agg) => agg is IFoundIndicesTracker);
bool trackerFound = (tracker != null);
if (!trackerFound)
{
tracker = new CountAggregator();
}
int count;
while ((count = _source.Next(XTableExtensions.DefaultBatchSize, cancellationToken)) != 0)
{
// Aggregate each row directly on the row index (already a small zero-based value)
XArray indices = indicesGetter();
for (int i = 0; i < _aggregators.Length; ++i)
{
_aggregators[i].Add(indices, values.Count);
}
if (!trackerFound)
{
tracker.Add(indices, values.Count);
}
}
// Figure out which rows had matches
ArraySelector foundValuesSelector = tracker.FoundIndices;
// Store the distinct count now that we know it
_distinctCount = foundValuesSelector.Count;
// Once the loop is done, get the distinct values and aggregation results
_columns[0].SetValues(values.Reselect(foundValuesSelector));
for (int i = 0; i < _aggregators.Length; ++i)
{
_columns[i + 1].SetValues(_aggregators[i].Values.Reselect(foundValuesSelector));
}
}
private void Convert()
{
// Close the row index writer
_rowIndexWriter.Dispose();
_rowIndexWriter = null;
// If we wrote any rows we need to convert...
if (_rowCountWritten > 0)
{
// Get the set of unique values and get rid of the value dictionary
XArray values = _dictionary.Values();
// Convert the indices previously written into raw values
Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(typeof(byte), typeof(int));
using (IColumnReader rowIndexReader = new PrimitiveArrayReader <byte>(_streamProvider.OpenRead(Path.Combine(_columnPath, RowIndexFileName))))
{
int rowCount = rowIndexReader.Count;
ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240);
while (page.Count > 0)
{
// Read an XArray of indices and convert to int[]
XArray rowIndices = converter(rowIndexReader.Read(page));
// Write the corresponding values
// Reselect is safe because 'values' are converted to a contiguous array
_valueWriter.Append(values.Reselect(ArraySelector.Map((int[])rowIndices.Array, rowIndices.Count)));
page = page.NextPage(rowCount, 10240);
}
}
}
// Remove the Dictionary (so future rows are streamed out as-is)
_dictionary = null;
// Delete the row index file
_streamProvider.Delete(Path.Combine(_columnPath, RowIndexFileName));
}
public Func <ArraySelector, XArray> SeekGetter()
{
return((selector) => _allValues.Reselect(selector));
}
