public async Task Ctor_MemoryPool_AllocatesFromPool(bool shouldLeaveOpen) { RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 100); using (MemoryStream stream = new MemoryStream()) { ArrowFileWriter writer = new ArrowFileWriter(stream, originalBatch.Schema); await writer.WriteRecordBatchAsync(originalBatch); await writer.WriteEndAsync(); stream.Position = 0; var memoryPool = new TestMemoryAllocator(); ArrowFileReader reader = new ArrowFileReader(stream, memoryPool, leaveOpen: shouldLeaveOpen); reader.ReadNextRecordBatch(); Assert.Equal(1, memoryPool.Statistics.Allocations); Assert.True(memoryPool.Statistics.BytesAllocated > 0); reader.Dispose(); if (shouldLeaveOpen) { Assert.True(stream.Position > 0); } else { Assert.Throws <ObjectDisposedException>(() => stream.Position); } } }
private async Task <int> Validate() { JsonFile jsonFile = await ParseJsonFile(); using FileStream arrowFileStream = ArrowFileInfo.OpenRead(); using ArrowFileReader reader = new ArrowFileReader(arrowFileStream); int batchCount = await reader.RecordBatchCountAsync(); if (batchCount != jsonFile.Batches.Count) { Console.WriteLine($"Incorrect batch count. JsonFile: {jsonFile.Batches.Count}, ArrowFile: {batchCount}"); return(-1); } Schema jsonFileSchema = CreateSchema(jsonFile.Schema); Schema arrowFileSchema = reader.Schema; SchemaComparer.Compare(jsonFileSchema, arrowFileSchema); for (int i = 0; i < batchCount; i++) { RecordBatch arrowFileRecordBatch = reader.ReadNextRecordBatch(); RecordBatch jsonFileRecordBatch = CreateRecordBatch(jsonFileSchema, jsonFile.Batches[i]); ArrowReaderVerifier.CompareBatches(jsonFileRecordBatch, arrowFileRecordBatch, strictCompare: false); } // ensure there are no more batches in the file if (reader.ReadNextRecordBatch() != null) { Console.WriteLine($"The ArrowFile has more RecordBatches than it should."); return(-1); } return(0); }
public async Task WritesEmptyFile() { RecordBatch originalBatch = TestData.CreateSampleRecordBatch(length: 1); var stream = new MemoryStream(); var writer = new ArrowFileWriter(stream, originalBatch.Schema); writer.WriteStart(); writer.WriteEnd(); stream.Position = 0; var reader = new ArrowFileReader(stream); int count = await reader.RecordBatchCountAsync(); Assert.Equal(0, count); RecordBatch readBatch = reader.ReadNextRecordBatch(); Assert.Null(readBatch); SchemaComparer.Compare(originalBatch.Schema, reader.Schema); }