public XArray Remap(XArray source, ref int[] remapArray) { // See if we have the remapping cached already ArraySelector cachedMapping; if (_cachedRemappings.TryGetValue(source.Selector, out cachedMapping)) { return(source.Reselect(cachedMapping)); } // Convert the BitVector to indices if we haven't yet (deferred to first column wanting values) if (!_indicesFound) { _indicesFound = true; Allocator.AllocateToSize(ref _indices, _count); int countFound = _vector.Page(_indices, ref _nextVectorIndex, _count); if (countFound != _count) { throw new InvalidOperationException($"RowRemapper found {countFound:n0} rows when {_count:n0} expected paging in Vector with {_vector.Count:n0} total matches up to index {_nextVectorIndex:n0}."); } } // Remap the outer selector XArray remapped = source.Select(ArraySelector.Map(_indices, _count), ref remapArray); // Cache the remapping _cachedRemappings[source.Selector] = remapped.Selector; return(remapped); }
private static void RoundTrip(string columnName, int[] array, int batchSize = 128) { XDatabaseContext context = new XDatabaseContext(); string columnPath = Path.Combine("VariableIntegerReaderWriterTests", columnName); string columnPrefix = Path.Combine(columnPath, "Vl"); context.StreamProvider.Delete(columnPath); Directory.CreateDirectory(columnPath); XArray values = XArray.All(array, array.Length); using (IColumnWriter writer = new VariableIntegerWriter(context.StreamProvider, columnPrefix)) { ArraySelector page = ArraySelector.All(0).NextPage(array.Length, batchSize); while (page.Count > 0) { writer.Append(values.Reselect(page)); page = page.NextPage(array.Length, batchSize); } } XArray returned = default(XArray); using (IColumnReader reader = new VariableIntegerReader(context.StreamProvider, columnPrefix, CachingOption.AsConfigured)) { returned = reader.Read(ArraySelector.All(array.Length)); } TableTestHarness.AssertAreEqual(values, returned, array.Length); context.StreamProvider.Delete(columnPath); }
public XArray Remap(XArray values, ArraySelector selector) { // Read row indices and convert to int[] XArray indexByteArray = _rowIndexReader.Read(selector); XArray indexIntArray = _rowIndexToIntConverter(indexByteArray); // Return the selected values return(values.Reselect(ArraySelector.Map((int[])indexIntArray.Array, indexIntArray.Count))); }
private void BuildSingleEnumColumnDictionary(CancellationToken cancellationToken) { XArray values = _keyColumns[0].ValuesGetter()(); Func <XArray> indicesGetter = _keyColumns[0].IndicesCurrentGetter(); // Find or construct an aggregator which can track which enum values ended up with any rows in the result IFoundIndicesTracker tracker = (IFoundIndicesTracker)_aggregators.FirstOrDefault((agg) => agg is IFoundIndicesTracker); bool trackerFound = (tracker != null); if (!trackerFound) { tracker = new CountAggregator(); } int count; while ((count = _source.Next(XTableExtensions.DefaultBatchSize, cancellationToken)) != 0) { // Aggregate each row directly on the row index (already a small zero-based value) XArray indices = indicesGetter(); for (int i = 0; i < _aggregators.Length; ++i) { _aggregators[i].Add(indices, values.Count); } if (!trackerFound) { tracker.Add(indices, values.Count); } } // Figure out which rows had matches ArraySelector foundValuesSelector = tracker.FoundIndices; // Store the distinct count now that we know it _distinctCount = foundValuesSelector.Count; // Once the loop is done, get the distinct values and aggregation results _columns[0].SetValues(values.Reselect(foundValuesSelector)); for (int i = 0; i < _aggregators.Length; ++i) { _columns[i + 1].SetValues(_aggregators[i].Values.Reselect(foundValuesSelector)); } }
private void Convert() { // Close the row index writer _rowIndexWriter.Dispose(); _rowIndexWriter = null; // If we wrote any rows we need to convert... if (_rowCountWritten > 0) { // Get the set of unique values and get rid of the value dictionary XArray values = _dictionary.Values(); // Convert the indices previously written into raw values Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(typeof(byte), typeof(int)); using (IColumnReader rowIndexReader = new PrimitiveArrayReader <byte>(_streamProvider.OpenRead(Path.Combine(_columnPath, RowIndexFileName)))) { int rowCount = rowIndexReader.Count; ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240); while (page.Count > 0) { // Read an XArray of indices and convert to int[] XArray rowIndices = converter(rowIndexReader.Read(page)); // Write the corresponding values // Reselect is safe because 'values' are converted to a contiguous array _valueWriter.Append(values.Reselect(ArraySelector.Map((int[])rowIndices.Array, rowIndices.Count))); page = page.NextPage(rowCount, 10240); } } } // Remove the Dictionary (so future rows are streamed out as-is) _dictionary = null; // Delete the row index file _streamProvider.Delete(Path.Combine(_columnPath, RowIndexFileName)); }
public Func <ArraySelector, XArray> SeekGetter() { return((selector) => _allValues.Reselect(selector)); }