public void Append(XArray xarray)
{
// If we already had too many values, we're just writing them out normally
if (_dictionary == null)
{
_valueWriter.Append(xarray);
_rowCountWritten += xarray.Count;
return;
}
// Otherwise, find the index of each value added
if (_dictionary.Add(xarray, ref _currentArrayIndices))
{
// If we're still under 256 values, write the indices
_rowIndexWriter.Append(XArray.All(_currentArrayIndices, xarray.Count));
_rowCountWritten += xarray.Count;
}
else
{
// If we went over 256 values, convert to writing the values directly
Convert();
_valueWriter.Append(xarray);
_rowCountWritten += xarray.Count;
return;
}
}
public void Append(XArray xarray)
{
// Write the values (without the null markers; we're writing those here)
_valueWriter.Append(xarray.WithoutNulls());
// Track the row count written so we know how many null=false values to write when we first see a null
_rowCountWritten += xarray.Count;
// If there are no nulls in this set and none previously, no null markers need to be written
if (!xarray.HasNulls && _nullWriter == null)
{
return;
}
if (_nullWriter == null)
{
// Check whether any rows in the set are actually null; the source may contain nulls but the filtered rows might not
bool areAnyNulls = false;
for (int i = 0; i < xarray.Count && !areAnyNulls; ++i)
{
areAnyNulls |= xarray.NullRows[xarray.Index(i)];
}
// If there are not actually any null rows in this set, don't write null output yet
if (!areAnyNulls)
{
return;
}
// Open a new file to write IsNull booleans
string nullsPath = Path.Combine(_columnPath, "Vn.b8.bin");
_nullWriter = new PrimitiveArrayWriter <bool>(_streamProvider.OpenWrite(nullsPath));
// Write false for every value so far
int previousCount = _rowCountWritten - xarray.Count;
Allocator.AllocateToSize(ref _falseArray, 1024);
for (int i = 0; i < previousCount; i += 1024)
{
int rowCount = Math.Min(1024, previousCount - i);
_nullWriter.Append(XArray.All(_falseArray, rowCount));
}
}
if (!xarray.HasNulls)
{
// If this xarray doesn't have any nulls, write false for every value in this page
Allocator.AllocateToSize(ref _falseArray, xarray.Count);
_nullWriter.Append(XArray.All(_falseArray, xarray.Count));
}
else
{
// Write the actual true/false values for this page
_nullWriter.Append(XArray.All(xarray.NullRows).Reselect(xarray.Selector));
}
}
private void Upconvert(XArray values)
{
if (WritingAsType == typeof(byte))
{
// Try to convert the new values to ushort
Func <XArray, XArray> ushortConverter = TypeConverterFactory.GetConverter(typeof(int), typeof(ushort));
XArray asUshort = ushortConverter(values);
// If none overflow, just upconvert to ushort
if (!asUshort.HasNulls)
{
Upconvert(typeof(ushort));
_writer.Append(asUshort);
return;
}
}
// If we were already ushort or new values overflowed ushort, upconvert to int
Upconvert(typeof(int));
_writer.Append(values);
}
private void Upconvert(Type toType)
{
// Close the current writer
_writer.Dispose();
_writer = null;
// Determine previous and new file paths
string columnValuesFullPath = PathForType(_columnPathPrefix, WritingAsType);
string columnConvertedFullPath = PathForType(_columnPathPrefix, toType);
// Build a writer for the larger type
IColumnWriter writer = BuildDirectWriter(_streamProvider, toType, columnConvertedFullPath);
// Convert already written values (if any)
if (_rowCountWritten > 0)
{
// Build a converter to convert the values
Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(WritingAsType, toType);
// Stream them in, convert them, and write them out
using (IColumnReader reader = TypeProviderFactory.TryGetColumnReader(_streamProvider, WritingAsType, columnValuesFullPath))
{
int rowCount = reader.Count;
ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240);
while (page.Count > 0)
{
XArray original = reader.Read(page);
XArray converted = converter(original);
writer.Append(converted);
page = page.NextPage(rowCount, 10240);
}
}
}
// Delete the original file
_streamProvider.Delete(columnValuesFullPath);
// Re-initialize for the new writer
WritingAsType = toType;
_writer = writer;
_converter = (toType == typeof(int) ? null : TypeConverterFactory.GetConverter(typeof(int), toType));
}
public void Append(XArray xarray)
{
_convertedValueWriter.Append(_converter(xarray));
}
