public void Append(XArray xarray) { // If we already had too many values, we're just writing them out normally if (_dictionary == null) { _valueWriter.Append(xarray); _rowCountWritten += xarray.Count; return; } // Otherwise, find the index of each value added if (_dictionary.Add(xarray, ref _currentArrayIndices)) { // If we're still under 256 values, write the indices _rowIndexWriter.Append(XArray.All(_currentArrayIndices, xarray.Count)); _rowCountWritten += xarray.Count; } else { // If we went over 256 values, convert to writing the values directly Convert(); _valueWriter.Append(xarray); _rowCountWritten += xarray.Count; return; } }
public void Append(XArray xarray) { // Write the values (without the null markers; we're writing those here) _valueWriter.Append(xarray.WithoutNulls()); // Track the row count written so we know how many null=false values to write when we first see a null _rowCountWritten += xarray.Count; // If there are no nulls in this set and none previously, no null markers need to be written if (!xarray.HasNulls && _nullWriter == null) { return; } if (_nullWriter == null) { // Check whether any rows in the set are actually null; the source may contain nulls but the filtered rows might not bool areAnyNulls = false; for (int i = 0; i < xarray.Count && !areAnyNulls; ++i) { areAnyNulls |= xarray.NullRows[xarray.Index(i)]; } // If there are not actually any null rows in this set, don't write null output yet if (!areAnyNulls) { return; } // Open a new file to write IsNull booleans string nullsPath = Path.Combine(_columnPath, "Vn.b8.bin"); _nullWriter = new PrimitiveArrayWriter <bool>(_streamProvider.OpenWrite(nullsPath)); // Write false for every value so far int previousCount = _rowCountWritten - xarray.Count; Allocator.AllocateToSize(ref _falseArray, 1024); for (int i = 0; i < previousCount; i += 1024) { int rowCount = Math.Min(1024, previousCount - i); _nullWriter.Append(XArray.All(_falseArray, rowCount)); } } if (!xarray.HasNulls) { // If this xarray doesn't have any nulls, write false for every value in this page Allocator.AllocateToSize(ref _falseArray, xarray.Count); _nullWriter.Append(XArray.All(_falseArray, xarray.Count)); } else { // Write the actual true/false values for this page _nullWriter.Append(XArray.All(xarray.NullRows).Reselect(xarray.Selector)); } }
private void Upconvert(XArray values) { if (WritingAsType == typeof(byte)) { // Try to convert the new values to ushort Func <XArray, XArray> ushortConverter = TypeConverterFactory.GetConverter(typeof(int), typeof(ushort)); XArray asUshort = ushortConverter(values); // If none overflow, just upconvert to ushort if (!asUshort.HasNulls) { Upconvert(typeof(ushort)); _writer.Append(asUshort); return; } } // If we were already ushort or new values overflowed ushort, upconvert to int Upconvert(typeof(int)); _writer.Append(values); }
private void Upconvert(Type toType) { // Close the current writer _writer.Dispose(); _writer = null; // Determine previous and new file paths string columnValuesFullPath = PathForType(_columnPathPrefix, WritingAsType); string columnConvertedFullPath = PathForType(_columnPathPrefix, toType); // Build a writer for the larger type IColumnWriter writer = BuildDirectWriter(_streamProvider, toType, columnConvertedFullPath); // Convert already written values (if any) if (_rowCountWritten > 0) { // Build a converter to convert the values Func <XArray, XArray> converter = TypeConverterFactory.GetConverter(WritingAsType, toType); // Stream them in, convert them, and write them out using (IColumnReader reader = TypeProviderFactory.TryGetColumnReader(_streamProvider, WritingAsType, columnValuesFullPath)) { int rowCount = reader.Count; ArraySelector page = ArraySelector.All(0).NextPage(rowCount, 10240); while (page.Count > 0) { XArray original = reader.Read(page); XArray converted = converter(original); writer.Append(converted); page = page.NextPage(rowCount, 10240); } } } // Delete the original file _streamProvider.Delete(columnValuesFullPath); // Re-initialize for the new writer WritingAsType = toType; _writer = writer; _converter = (toType == typeof(int) ? null : TypeConverterFactory.GetConverter(typeof(int), toType)); }
public void Append(XArray xarray) { _convertedValueWriter.Append(_converter(xarray)); }