public virtual bool HasNext() => _avroReader.HasNext();
// Note - offset is with respect to buffer.
private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count)
{
ValidateReadParameters(buffer, offset, count);
int remainingBytes = _bufferLength - _bufferOffset;
// We have enough bytes in the buffer and don't need to read the next Record.
if (count <= remainingBytes)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: count);
_bufferOffset += count;
return(count);
}
// Copy remaining buffer
if (remainingBytes > 0)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: remainingBytes);
_bufferOffset += remainingBytes;
return(remainingBytes);
}
// Reset _bufferOffset, _bufferLength, and remainingBytes
_bufferOffset = 0;
_bufferLength = 0;
remainingBytes = 0;
// We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream.
if (!_avroReader.HasNext())
{
return(0);
}
// We need to keep getting the next record until we get a data record.
while (remainingBytes == 0)
{
// Get next Record.
Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false);
switch (record["$schema"])
{
// Data Record
case Constants.QuickQuery.DataRecordName:
record.TryGetValue(Constants.QuickQuery.Data, out object byteObject);
if (byteObject == null)
{
throw new InvalidOperationException($"Avro data record is missing {Constants.QuickQuery.Data} property");
}
byte[] bytes = (byte[])byteObject;
// Return the buffer if it is not null and not big enough.
if (_buffer != null && _buffer.Length < bytes.Length)
{
ArrayPool <byte> .Shared.Return(_buffer, clearArray : true);
}
// Rent a new buffer if it is null or not big enough.
if (_buffer == null || _buffer.Length < bytes.Length)
{
_buffer = ArrayPool <byte> .Shared.Rent(Math.Max(4 * Constants.MB, bytes.Length));
}
Array.Copy(
sourceArray: bytes,
sourceIndex: 0,
destinationArray: _buffer,
destinationIndex: 0,
length: bytes.Length);
_bufferLength = bytes.Length;
// Don't remove this reset, it is used in the final array copy below.
remainingBytes = bytes.Length;
break;
// Progress Record
case Constants.QuickQuery.ProgressRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress);
if (progress == null)
{
throw new InvalidOperationException($"Avro progress record is mssing {Constants.QuickQuery.BytesScanned} property");
}
_progressHandler.Report((long)progress);
}
break;
// Error Record
case Constants.QuickQuery.ErrorRecordName:
ProcessErrorRecord(record);
break;
// End Record
case Constants.QuickQuery.EndRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress);
if (progress == null)
{
throw new InvalidOperationException($"Avro end record is missing {Constants.QuickQuery.TotalBytes} property");
}
_progressHandler.Report((long)progress);
}
return(0);
}
}
int length = Math.Min(count, remainingBytes);
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: length);
_bufferOffset += length;
return(length);
}
// Note - offset is with respect to buffer.
private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count)
{
ValidateReadParameters(buffer, offset, count);
int remainingBytes = _bufferLength - _bufferOffset;
// We have enough bytes in the buffer and don't need to read the next Record.
if (count <= remainingBytes)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: count);
_bufferOffset += count;
return(count);
}
// Copy remaining buffer
if (remainingBytes > 0)
{
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: remainingBytes);
_bufferOffset += remainingBytes;
return(remainingBytes);
}
// Reset _bufferOffset, _bufferLength, and remainingBytes
_bufferOffset = 0;
_bufferLength = 0;
remainingBytes = 0;
// We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream.
// TODO what to do in this case? If we return 0, we are indicating the end of stream
if (!_avroReader.HasNext())
{
return(0);
}
// We need to keep getting the next record until we get a data record.
while (remainingBytes == 0)
{
// Get next Record.
Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false);
switch (record["$schema"])
{
// Data Record
case Constants.QuickQuery.DataRecordName:
record.TryGetValue(Constants.QuickQuery.Data, out object byteObject);
byte[] bytes = (byte[])byteObject;
Array.Copy(
sourceArray: bytes,
sourceIndex: 0,
destinationArray: _buffer,
destinationIndex: 0,
length: bytes.Length);
_bufferLength = bytes.Length;
// Don't remove this reset, it is used in the final array copy below.
remainingBytes = bytes.Length;
break;
// Progress Record
case Constants.QuickQuery.ProgressRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress);
_progressHandler.Report((long)progress);
}
break;
// Error Record
case Constants.QuickQuery.ErrorRecordName:
ProcessErrorRecord(record);
break;
// End Record
case Constants.QuickQuery.EndRecordName:
if (_progressHandler != default)
{
record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress);
_progressHandler.Report((long)progress);
}
return(0);
}
}
int length = Math.Min(count, remainingBytes);
Array.Copy(
sourceArray: _buffer,
sourceIndex: _bufferOffset,
destinationArray: buffer,
destinationIndex: offset,
length: length);
_bufferOffset += length;
return(length);
}
