private static void RoundTrip(string columnName, int[] array, int batchSize = 128)
{
XDatabaseContext context = new XDatabaseContext();
string columnPath = Path.Combine("VariableIntegerReaderWriterTests", columnName);
string columnPrefix = Path.Combine(columnPath, "Vl");
context.StreamProvider.Delete(columnPath);
Directory.CreateDirectory(columnPath);
XArray values = XArray.All(array, array.Length);
using (IColumnWriter writer = new VariableIntegerWriter(context.StreamProvider, columnPrefix))
{
ArraySelector page = ArraySelector.All(0).NextPage(array.Length, batchSize);
while (page.Count > 0)
{
writer.Append(values.Reselect(page));
page = page.NextPage(array.Length, batchSize);
}
}
XArray returned = default(XArray);
using (IColumnReader reader = new VariableIntegerReader(context.StreamProvider, columnPrefix, CachingOption.AsConfigured))
{
returned = reader.Read(ArraySelector.All(array.Length));
}
TableTestHarness.AssertAreEqual(values, returned, array.Length);
context.StreamProvider.Delete(columnPath);
}
public String8Raw ReadRaw(ArraySelector selector)
{
if (selector.Equals(_currentRaw.Selector))
{
return(_currentRaw);
}
bool includesFirstString = (selector.StartIndexInclusive == 0);
_currentRaw.Selector = selector;
// Read the string positions
_currentRaw.Positions = _positionsReader.Read(ArraySelector.All(Count).Slice((includesFirstString ? 0 : selector.StartIndexInclusive - 1), selector.EndIndexExclusive));
if (_currentRaw.Positions.Selector.Indices != null)
{
throw new NotImplementedException("String8TypeProvider requires positions to be read contiguously.");
}
int[] positionArray = (int[])_currentRaw.Positions.Array;
// Get the full byte range of all of the strings
int firstStringStart = (includesFirstString ? 0 : positionArray[_currentRaw.Positions.Index(0)]);
int lastStringEnd = positionArray[_currentRaw.Positions.Index(_currentRaw.Positions.Count - 1)];
// Read the raw string bytes
_currentRaw.Bytes = _bytesReader.Read(ArraySelector.All(int.MaxValue).Slice(firstStringStart, lastStringEnd));
if (_currentRaw.Bytes.Selector.Indices != null)
{
throw new NotImplementedException("String8TypeProvider requires positions to be read contiguously.");
}
return(_currentRaw);
}
private XArray ReadIndices(ArraySelector selector)
{
Allocator.AllocateToSize(ref _resultArray, selector.Count);
// Read all string positions
XArray positions = _positionsReader.Read(ArraySelector.All(_positionsReader.Count));
int[] positionArray = (int[])positions.Array;
// Read all raw string bytes
XArray bytes = _bytesReader.Read(ArraySelector.All(_bytesReader.Count));
byte[] textArray = (byte[])bytes.Array;
// Update the String8 array to point to them
for (int i = 0; i < selector.Count; ++i)
{
int rowIndex = selector.Index(i);
int valueStart = (rowIndex == 0 ? 0 : positionArray[rowIndex - 1]);
int valueEnd = positionArray[rowIndex];
_resultArray[i] = new String8(textArray, valueStart, valueEnd - valueStart);
}
// Cache the xarray and return it
_currentArray = XArray.All(_resultArray, selector.Count);
_currentSelector = selector;
return(_currentArray);
}
public XArray Read(ArraySelector selector)
{
if (selector.Indices != null)
{
throw new NotImplementedException();
}
// Return the previous xarray if re-requested
if (selector.Equals(_currentSelector))
{
return(_currentArray);
}
// Allocate the result array
Allocator.AllocateToSize(ref _array, selector.Count);
// Read items in pages of 64k
int byteStart = _bytesPerItem * selector.StartIndexInclusive;
int byteEnd = _bytesPerItem * selector.EndIndexExclusive;
int bytesRead = 0;
for (int currentByteIndex = byteStart; currentByteIndex < byteEnd; currentByteIndex += ReadPageSize)
{
int currentByteEnd = Math.Min(byteEnd, currentByteIndex + ReadPageSize);
XArray bytexarray = _byteReader.Read(ArraySelector.All(int.MaxValue).Slice(currentByteIndex, currentByteEnd));
Buffer.BlockCopy(bytexarray.Array, 0, _array, bytesRead, bytexarray.Count);
bytesRead += currentByteEnd - currentByteIndex;
}
// Cache and return the current xarray
_currentArray = XArray.All(_array, selector.Count);
_currentSelector = selector;
return(_currentArray);
}
public int Next(int desiredCount, CancellationToken cancellationToken)
{
long countToReturn = Math.Min(_rowCount - _countEnumerated, desiredCount);
if (countToReturn == 0)
{
return(0);
}
_currentEnumerateSelector = _currentEnumerateSelector.NextPage(_arrayRowCount, (int)countToReturn);
if (_currentEnumerateSelector.Count == 0)
{
_currentEnumerateSelector = ArraySelector.All(_arrayRowCount).Slice(0, 0);
_currentEnumerateSelector = _currentEnumerateSelector.NextPage(_arrayRowCount, (int)countToReturn);
}
_currentSelector = _currentEnumerateSelector;
for (int i = 0; i < _columns.Count; ++i)
{
_columns[i].SetSelector(_currentEnumerateSelector);
}
CurrentRowCount = _currentEnumerateSelector.Count;
_countEnumerated += _currentEnumerateSelector.Count;
return(CurrentRowCount);
}
public override void Reset()
{
_enumerateSelector = ArraySelector.All(0);
_columns[0].SetSelector(_enumerateSelector);
_columns[1].SetSelector(_enumerateSelector);
}
public CachedColumnReader(IColumnReader inner)
{
using (inner)
{
_column = inner.Read(ArraySelector.All(inner.Count));
}
}
public SchemaTransformer(IXTable source) : base(source)
{
_columns = new ArrayColumn[2];
_columns[0] = new ArrayColumn(XArray.All(_source.Columns.Select((col) => col.ColumnDetails.Name).ToArray()), new ColumnDetails("Name", typeof(string)));
_columns[1] = new ArrayColumn(XArray.All(_source.Columns.Select((col) => col.ColumnDetails.Type.Name.ToString()).ToArray()), new ColumnDetails("Type", typeof(string)));
_enumerateSelector = ArraySelector.All(0);
}
public void Reset()
{
_currentEnumerateSelector = ArraySelector.All(_distinctCount).Slice(0, 0);
for (int i = 0; i < _columns.Length; ++i)
{
_columns[i].SetSelector(_currentEnumerateSelector);
}
}
public void Reset()
{
_countEnumerated = 0;
_currentEnumerateSelector = ArraySelector.All(_arrayRowCount).Slice(0, 0);
for (int i = 0; i < _columns.Count; ++i)
{
_columns[i].SetSelector(_currentEnumerateSelector);
}
}
// Values returns the set of distinct values themselves
public XArray Values()
{
// Read the values (if we haven't previously)
if (_allValues.Array == null)
{
_allValues = _valueReader.Read(ArraySelector.All(_valueReader.Count));
if (_allValues.Selector.Indices != null || _allValues.Selector.StartIndexInclusive != 0)
{
throw new InvalidOperationException("EnumColumnReader values reader must read values contiguously.");
}
}
return(_allValues);
}
private void BuildJoinDictionary(CancellationToken cancellationToken)
{
// Validate the RHS is a seekable table (only on build, so that Suggest doesn't fail)
ISeekableXTable joinToSource = _joinToSource as ISeekableXTable;
if (joinToSource == null)
{
throw new ArgumentException($"Join requires a single built Binary Table as the right side table.");
}
XArray allJoinToValues = _joinToSeekGetter(ArraySelector.All(joinToSource.Count));
_joinDictionary = (IJoinDictionary)Allocator.ConstructGenericOf(typeof(JoinDictionary <>), _joinColumnType, allJoinToValues.Count);
_joinDictionary.Add(allJoinToValues, 0);
}
public void Sampler_Basics()
{
Random r = new Random(8);
ArraySelector all = ArraySelector.All(10240);
int[] eighthArray = null;
ArraySelector eighth = Sampler.Eighth(all, r, ref eighthArray);
AssertClose(all.Count / 8, eighth.Count, 0.2f);
int[] sixtyfourthArray = null;
ArraySelector sixtyfourth = Sampler.Eighth(eighth, r, ref sixtyfourthArray);
AssertClose(eighth.Count / 8, sixtyfourth.Count, 0.2f);
}
public void Sample()
{
Random r = new Random(8);
ArraySelector all = ArraySelector.All(10240);
int[] eighthArray = null;
int[] sixtyfourthArray = null;
using (Benchmarker b = new Benchmarker($"Sampler.Eighth", DefaultMeasureMilliseconds))
{
b.Measure("Sampler.Eighth", all.Count, () =>
{
ArraySelector eighth = Sampler.Eighth(all, r, ref eighthArray);
ArraySelector sixtyfourth = Sampler.Eighth(eighth, r, ref sixtyfourthArray);
return(sixtyfourth.Count);
});
}
}
private void Upconvert(Type toType)
{
// Close the current writer
_writer.Dispose();
_writer = null;
// Determine previous and new file paths
string columnValuesFullPath = PathForType(_columnPathPrefix, WritingAsType);
string columnConvertedFullPath = PathForType(_columnPathPrefix, toType);
// Build a writer for the larger type
IColumnWriter writer = BuildDirectWriter(_streamProvider, toType, columnConvertedFullPath);
// Convert already written values (if any)
if (_rowCountWritten > 0)
{
// Build a converter to convert the values
Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(WritingAsType, toType);
// Stream them in, convert them, and write them out
using (IColumnReader reader = TypeProviderFactory.TryGetColumnReader(_streamProvider, WritingAsType, columnValuesFullPath))
{
int rowCount = reader.Count;
ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240);
while (page.Count > 0)
{
XArray original = reader.Read(page);
XArray converted = converter(original);
writer.Append(converted);
page = page.NextPage(rowCount, 10240);
}
}
}
// Delete the original file
_streamProvider.Delete(columnValuesFullPath);
// Re-initialize for the new writer
WritingAsType = toType;
_writer = writer;
_converter = (toType == typeof(int) ? null : TypeConverterFactory.GetConverter(typeof(int), toType));
}
private void Convert()
{
// Close the row index writer
_rowIndexWriter.Dispose();
_rowIndexWriter = null;
// If we wrote any rows we need to convert...
if (_rowCountWritten > 0)
{
// Get the set of unique values and get rid of the value dictionary
XArray values = _dictionary.Values();
// Convert the indices previously written into raw values
Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(typeof(byte), typeof(int));
using (IColumnReader rowIndexReader = new PrimitiveArrayReader <byte>(_streamProvider.OpenRead(Path.Combine(_columnPath, RowIndexFileName))))
{
int rowCount = rowIndexReader.Count;
ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240);
while (page.Count > 0)
{
// Read an XArray of indices and convert to int[]
XArray rowIndices = converter(rowIndexReader.Read(page));
// Write the corresponding values
// Reselect is safe because 'values' are converted to a contiguous array
_valueWriter.Append(values.Reselect(ArraySelector.Map((int[])rowIndices.Array, rowIndices.Count)));
page = page.NextPage(rowCount, 10240);
}
}
}
// Remove the Dictionary (so future rows are streamed out as-is)
_dictionary = null;
// Delete the row index file
_streamProvider.Delete(Path.Combine(_columnPath, RowIndexFileName));
}
public void Reset()
{
// Mark our current position (nothing read yet)
_currentEnumerateSelector = ArraySelector.All(Count).Slice(0, 0);
}
