public async Task Ctor_MemoryPool_AllocatesFromPool(bool shouldLeaveOpen)
{
RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
using (MemoryStream stream = new MemoryStream())
{
ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch.Schema);
await writer.WriteRecordBatchAsync(originalBatch);
await writer.WriteEndAsync();
stream.Position = 0;
var memoryPool = new TestMemoryAllocator();
ArrowFileReader reader = new ArrowFileReader(stream, memoryPool, leaveOpen: shouldLeaveOpen);
reader.ReadNextRecordBatch();
Assert.Equal(1, memoryPool.Statistics.Allocations);
Assert.True(memoryPool.Statistics.BytesAllocated > 0);
reader.Dispose();
if (shouldLeaveOpen)
{
Assert.True(stream.Position > 0);
}
else
{
Assert.Throws <ObjectDisposedException>(() => stream.Position);
}
}
}
private async Task <int> Validate()
{
JsonFile jsonFile = await ParseJsonFile();
using FileStream arrowFileStream = ArrowFileInfo.OpenRead();
using ArrowFileReader reader = new ArrowFileReader(arrowFileStream);
int batchCount = await reader.RecordBatchCountAsync();
if (batchCount != jsonFile.Batches.Count)
{
Console.WriteLine($"Incorrect batch count. JsonFile: {jsonFile.Batches.Count}, ArrowFile: {batchCount}");
return(-1);
}
Schema jsonFileSchema = CreateSchema(jsonFile.Schema);
Schema arrowFileSchema = reader.Schema;
SchemaComparer.Compare(jsonFileSchema, arrowFileSchema);
for (int i = 0; i < batchCount; i++)
{
RecordBatch arrowFileRecordBatch = reader.ReadNextRecordBatch();
RecordBatch jsonFileRecordBatch = CreateRecordBatch(jsonFileSchema, jsonFile.Batches[i]);
ArrowReaderVerifier.CompareBatches(jsonFileRecordBatch, arrowFileRecordBatch, strictCompare: false);
}
// ensure there are no more batches in the file
if (reader.ReadNextRecordBatch() != null)
{
Console.WriteLine($"The ArrowFile has more RecordBatches than it should.");
return(-1);
}
return(0);
}
public async Task WritesEmptyFile()
{
RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1);
var stream = new MemoryStream();
var writer = new ArrowFileWriter(stream, originalBatch.Schema);
writer.WriteStart();
writer.WriteEnd();
stream.Position = 0;
var reader = new ArrowFileReader(stream);
int count = await reader.RecordBatchCountAsync();
Assert.Equal(0, count);
RecordBatch readBatch = reader.ReadNextRecordBatch();
Assert.Null(readBatch);
SchemaComparer.Compare(originalBatch.Schema, reader.Schema);
}