public async Task TestReadMultipleRecordBatchAsync()
{
RecordBatch originalBatch1 = TestData.CreateSampleRecordBatch(length: 100);
RecordBatch originalBatch2 = TestData.CreateSampleRecordBatch(length: 50);
using (MemoryStream stream = new MemoryStream())
{
ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch1.Schema);
await writer.WriteRecordBatchAsync(originalBatch1);
await writer.WriteRecordBatchAsync(originalBatch2);
await writer.WriteFooterAsync();
stream.Position = 0;
// the recordbatches by index are in reverse order - back to front.
// TODO: is this a bug??
ArrowFileReader reader = new ArrowFileReader(stream);
RecordBatch readBatch1 = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(originalBatch2, readBatch1);
RecordBatch readBatch2 = await reader.ReadRecordBatchAsync(1);
ArrowReaderVerifier.CompareBatches(originalBatch1, readBatch2);
// now read the first again, for random access
RecordBatch readBatch3 = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(originalBatch2, readBatch3);
}
}
public async Task TestReadMultipleRecordBatchAsync()
{
RecordBatch originalBatch1 = TestData.CreateSampleRecordBatch(length: 100);
RecordBatch originalBatch2 = TestData.CreateSampleRecordBatch(length: 50);
using (MemoryStream stream = new MemoryStream())
{
ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch1.Schema);
await writer.WriteRecordBatchAsync(originalBatch1);
await writer.WriteRecordBatchAsync(originalBatch2);
await writer.WriteEndAsync();
stream.Position = 0;
ArrowFileReader reader = new ArrowFileReader(stream);
RecordBatch readBatch1 = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(originalBatch1, readBatch1);
RecordBatch readBatch2 = await reader.ReadRecordBatchAsync(1);
ArrowReaderVerifier.CompareBatches(originalBatch2, readBatch2);
// now read the first again, for random access
RecordBatch readBatch3 = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(originalBatch1, readBatch3);
}
}
public async Task WritesFooterAlignedMulitpleOf8()
{
RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100);
var stream = new MemoryStream();
var writer = new ArrowFileWriter(
stream,
originalBatch.Schema,
leaveOpen: true,
// use WriteLegacyIpcFormat, which only uses a 4-byte length prefix
// which causes the length prefix to not be 8-byte aligned by default
new IpcOptions()
{
WriteLegacyIpcFormat = true
});
await writer.WriteRecordBatchAsync(originalBatch);
await writer.WriteEndAsync();
stream.Position = 0;
var reader = new ArrowFileReader(stream);
int count = await reader.RecordBatchCountAsync();
Assert.Equal(1, count);
RecordBatch readBatch = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(originalBatch, readBatch);
}
private async Task ValidateRecordBatchFile(Stream stream, RecordBatch recordBatch)
{
var reader = new ArrowFileReader(stream);
int count = await reader.RecordBatchCountAsync();
Assert.Equal(1, count);
RecordBatch readBatch = await reader.ReadRecordBatchAsync(0);
ArrowReaderVerifier.CompareBatches(recordBatch, readBatch);
}
