public async Task Tests()
{
List <TestCase> testCases = new List <TestCase>
{
new TestCase("test_null_0.avro", o => Assert.IsNull(o)), // null
new TestCase("test_null_1.avro", o => Assert.AreEqual(true, (bool)o)), // bool
new TestCase("test_null_2.avro", o => Assert.AreEqual("adsfasdf09809dsf-=adsf", (string)o)), // string
new TestCase("test_null_3.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("12345abcd"), (byte[])o)), // byte[]
new TestCase("test_null_4.avro", o => Assert.AreEqual(1234, (int)o)), // int
new TestCase("test_null_5.avro", o => Assert.AreEqual(1234L, (long)o)), // long
new TestCase("test_null_6.avro", o => Assert.AreEqual(1234.0, (float)o)), // float
new TestCase("test_null_7.avro", o => Assert.AreEqual(1234.0, (double)o)), // fouble
// Not supported today.
//new TestCase("test_null_8.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("B"), (byte[])o)), // fixed
new TestCase("test_null_9.avro", o => Assert.AreEqual("B", (string)o)), // enum
// Not supported today.
// new TestCase("test_null_10.avro", o => Assert.AreEqual(new List<long>() { 1, 2, 3 }, (List<long>)o)), // array
new TestCase("test_null_11.avro", o => Assert.AreEqual(
new Dictionary <string, int>()
{
{ "a", 1 }, { "b", 3 }, { "c", 2 }
}, (Dictionary <string, object>)o)), // dictionary
new TestCase("test_null_12.avro", o => Assert.IsNull(o)), // union
new TestCase("test_null_13.avro", o => // record
{
Dictionary <string, object> expected = new Dictionary <string, object>()
{
{ "$schema", "Test" }, { "f", 5 }
};
Dictionary <string, object> actual = (Dictionary <string, object>)o;
Assert.AreEqual(expected.Count, actual.Count);
foreach (KeyValuePair <string, object> keyValuePair in actual)
{
Assert.AreEqual(expected[keyValuePair.Key], keyValuePair.Value);
}
})
};
foreach (TestCase testCase in testCases)
{
// Arrange
using FileStream stream = File.OpenRead(
$"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}");
AvroReader avroReader = new AvroReader(stream);
// Act
object o = await avroReader.Next(async : true).ConfigureAwait(false);
testCase.Predicate(o);
}
}
public virtual async Task <BlobChangeFeedEvent> Next(
bool async,
CancellationToken cancellationToken = default)
{
Dictionary <string, object> result;
if (!HasNext())
{
return(null);
}
result = (Dictionary <string, object>) await _avroReader.Next(async, cancellationToken).ConfigureAwait(false);
BlockOffset = _avroReader.BlockOffset;
EventIndex = _avroReader.ObjectIndex;
return(new BlobChangeFeedEvent(result));
}
// Note - offset is with respect to buffer.
private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count)
{
ValidateReadParameters(buffer, offset, count);
int remainingBytes = _bufferLength - _bufferOffset;
// We have enough bytes in the buffer and don't need to read the next Record.
if (count <= remainingBytes)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: count);
_bufferOffset += count;
return(count);
}
// Copy remaining buffer
if (remainingBytes > 0)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: remainingBytes);
_bufferOffset += remainingBytes;
return(remainingBytes);
}
// Reset _bufferOffset, _bufferLength, and remainingBytes
_bufferOffset = 0;
_bufferLength = 0;
remainingBytes = 0;
// We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream.
if (!_avroReader.HasNext())
{
return(0);
}
// We need to keep getting the next record until we get a data record.
while (remainingBytes == 0)
{
// Get next Record.
Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false);
switch (record["$schema"])
{
// Data Record
case Constants.QuickQuery.DataRecordName:
record.TryGetValue(Constants.QuickQuery.Data, out object byteObject);
if (byteObject == null)
{
throw new InvalidOperationException($"Avro data record is missing {Constants.QuickQuery.Data} property");
}
byte[] bytes = (byte[])byteObject;
// Return the buffer if it is not null and not big enough.
if (_buffer != null && _buffer.Length < bytes.Length)
{
ArrayPool <byte> .Shared.Return(_buffer, clearArray : true);
}
// Rent a new buffer if it is null or not big enough.
if (_buffer == null || _buffer.Length < bytes.Length)
{
_buffer = ArrayPool <byte> .Shared.Rent(Math.Max(4 * Constants.MB, bytes.Length));
}
Array.Copy(
sourceArray: bytes,
sourceIndex: 0,
destinationArray: _buffer,
destinationIndex: 0,
length: bytes.Length);
_bufferLength = bytes.Length;
// Don't remove this reset, it is used in the final array copy below.
remainingBytes = bytes.Length;
break;
// Progress Record
case Constants.QuickQuery.ProgressRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress);
if (progress == null)
{
throw new InvalidOperationException($"Avro progress record is mssing {Constants.QuickQuery.BytesScanned} property");
}
_progressHandler.Report((long)progress);
}
break;
// Error Record
case Constants.QuickQuery.ErrorRecordName:
ProcessErrorRecord(record);
break;
// End Record
case Constants.QuickQuery.EndRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress);
if (progress == null)
{
throw new InvalidOperationException($"Avro end record is missing {Constants.QuickQuery.TotalBytes} property");
}
_progressHandler.Report((long)progress);
}
return(0);
}
}
int length = Math.Min(count, remainingBytes);
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: length);
_bufferOffset += length;
return(length);
}
// Note - offset is with respect to buffer.
private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count)
{
ValidateReadParameters(buffer, offset, count);
int remainingBytes = _bufferLength - _bufferOffset;
// We have enough bytes in the buffer and don't need to read the next Record.
if (count <= remainingBytes)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: count);
_bufferOffset += count;
return(count);
}
// Copy remaining buffer
if (remainingBytes > 0)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: remainingBytes);
_bufferOffset += remainingBytes;
return(remainingBytes);
}
// Reset _bufferOffset, _bufferLength, and remainingBytes
_bufferOffset = 0;
_bufferLength = 0;
remainingBytes = 0;
// We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream.
// TODO what to do in this case? If we return 0, we are indicating the end of stream
if (!_avroReader.HasNext())
{
return(0);
}
// We need to keep getting the next record until we get a data record.
while (remainingBytes == 0)
{
// Get next Record.
Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false);
switch (record["$schema"])
{
// Data Record
case Constants.QuickQuery.DataRecordName:
record.TryGetValue(Constants.QuickQuery.Data, out object byteObject);
byte[] bytes = (byte[])byteObject;
Array.Copy(
sourceArray: bytes,
sourceIndex: 0,
destinationArray: _buffer,
destinationIndex: 0,
length: bytes.Length);
_bufferLength = bytes.Length;
// Don't remove this reset, it is used in the final array copy below.
remainingBytes = bytes.Length;
break;
// Progress Record
case Constants.QuickQuery.ProgressRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress);
_progressHandler.Report((long)progress);
}
break;
// Error Record
case Constants.QuickQuery.ErrorRecordName:
ProcessErrorRecord(record);
break;
// End Record
case Constants.QuickQuery.EndRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress);
_progressHandler.Report((long)progress);
}
return(0);
}
}
int length = Math.Min(count, remainingBytes);
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: length);
_bufferOffset += length;
return(length);
}
