public virtual bool HasNext() => _avroReader.HasNext();
// Note - offset is with respect to buffer. private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count) { ValidateReadParameters(buffer, offset, count); int remainingBytes = _bufferLength - _bufferOffset; // We have enough bytes in the buffer and don't need to read the next Record. if (count <= remainingBytes) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: count); _bufferOffset += count; return(count); } // Copy remaining buffer if (remainingBytes > 0) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: remainingBytes); _bufferOffset += remainingBytes; return(remainingBytes); } // Reset _bufferOffset, _bufferLength, and remainingBytes _bufferOffset = 0; _bufferLength = 0; remainingBytes = 0; // We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream. if (!_avroReader.HasNext()) { return(0); } // We need to keep getting the next record until we get a data record. while (remainingBytes == 0) { // Get next Record. Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false); switch (record["$schema"]) { // Data Record case Constants.QuickQuery.DataRecordName: record.TryGetValue(Constants.QuickQuery.Data, out object byteObject); if (byteObject == null) { throw new InvalidOperationException($"Avro data record is missing {Constants.QuickQuery.Data} property"); } byte[] bytes = (byte[])byteObject; // Return the buffer if it is not null and not big enough. if (_buffer != null && _buffer.Length < bytes.Length) { ArrayPool <byte> .Shared.Return(_buffer, clearArray : true); } // Rent a new buffer if it is null or not big enough. if (_buffer == null || _buffer.Length < bytes.Length) { _buffer = ArrayPool <byte> .Shared.Rent(Math.Max(4 * Constants.MB, bytes.Length)); } Array.Copy( sourceArray: bytes, sourceIndex: 0, destinationArray: _buffer, destinationIndex: 0, length: bytes.Length); _bufferLength = bytes.Length; // Don't remove this reset, it is used in the final array copy below. remainingBytes = bytes.Length; break; // Progress Record case Constants.QuickQuery.ProgressRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress); if (progress == null) { throw new InvalidOperationException($"Avro progress record is mssing {Constants.QuickQuery.BytesScanned} property"); } _progressHandler.Report((long)progress); } break; // Error Record case Constants.QuickQuery.ErrorRecordName: ProcessErrorRecord(record); break; // End Record case Constants.QuickQuery.EndRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress); if (progress == null) { throw new InvalidOperationException($"Avro end record is missing {Constants.QuickQuery.TotalBytes} property"); } _progressHandler.Report((long)progress); } return(0); } } int length = Math.Min(count, remainingBytes); Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: length); _bufferOffset += length; return(length); }
// Note - offset is with respect to buffer. private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count) { ValidateReadParameters(buffer, offset, count); int remainingBytes = _bufferLength - _bufferOffset; // We have enough bytes in the buffer and don't need to read the next Record. if (count <= remainingBytes) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: count); _bufferOffset += count; return(count); } // Copy remaining buffer if (remainingBytes > 0) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: remainingBytes); _bufferOffset += remainingBytes; return(remainingBytes); } // Reset _bufferOffset, _bufferLength, and remainingBytes _bufferOffset = 0; _bufferLength = 0; remainingBytes = 0; // We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream. // TODO what to do in this case? If we return 0, we are indicating the end of stream if (!_avroReader.HasNext()) { return(0); } // We need to keep getting the next record until we get a data record. while (remainingBytes == 0) { // Get next Record. Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false); switch (record["$schema"]) { // Data Record case Constants.QuickQuery.DataRecordName: record.TryGetValue(Constants.QuickQuery.Data, out object byteObject); byte[] bytes = (byte[])byteObject; Array.Copy( sourceArray: bytes, sourceIndex: 0, destinationArray: _buffer, destinationIndex: 0, length: bytes.Length); _bufferLength = bytes.Length; // Don't remove this reset, it is used in the final array copy below. remainingBytes = bytes.Length; break; // Progress Record case Constants.QuickQuery.ProgressRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress); _progressHandler.Report((long)progress); } break; // Error Record case Constants.QuickQuery.ErrorRecordName: ProcessErrorRecord(record); break; // End Record case Constants.QuickQuery.EndRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress); _progressHandler.Report((long)progress); } return(0); } } int length = Math.Min(count, remainingBytes); Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: length); _bufferOffset += length; return(length); }