public async Task Tests() { List <TestCase> testCases = new List <TestCase> { new TestCase("test_null_0.avro", o => Assert.IsNull(o)), // null new TestCase("test_null_1.avro", o => Assert.AreEqual(true, (bool)o)), // bool new TestCase("test_null_2.avro", o => Assert.AreEqual("adsfasdf09809dsf-=adsf", (string)o)), // string new TestCase("test_null_3.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("12345abcd"), (byte[])o)), // byte[] new TestCase("test_null_4.avro", o => Assert.AreEqual(1234, (int)o)), // int new TestCase("test_null_5.avro", o => Assert.AreEqual(1234L, (long)o)), // long new TestCase("test_null_6.avro", o => Assert.AreEqual(1234.0, (float)o)), // float new TestCase("test_null_7.avro", o => Assert.AreEqual(1234.0, (double)o)), // fouble // Not supported today. //new TestCase("test_null_8.avro", o => Assert.AreEqual(Encoding.UTF8.GetBytes("B"), (byte[])o)), // fixed new TestCase("test_null_9.avro", o => Assert.AreEqual("B", (string)o)), // enum // Not supported today. // new TestCase("test_null_10.avro", o => Assert.AreEqual(new List<long>() { 1, 2, 3 }, (List<long>)o)), // array new TestCase("test_null_11.avro", o => Assert.AreEqual( new Dictionary <string, int>() { { "a", 1 }, { "b", 3 }, { "c", 2 } }, (Dictionary <string, object>)o)), // dictionary new TestCase("test_null_12.avro", o => Assert.IsNull(o)), // union new TestCase("test_null_13.avro", o => // record { Dictionary <string, object> expected = new Dictionary <string, object>() { { "$schema", "Test" }, { "f", 5 } }; Dictionary <string, object> actual = (Dictionary <string, object>)o; Assert.AreEqual(expected.Count, actual.Count); foreach (KeyValuePair <string, object> keyValuePair in actual) { Assert.AreEqual(expected[keyValuePair.Key], keyValuePair.Value); } }) }; foreach (TestCase testCase in testCases) { // Arrange using FileStream stream = File.OpenRead( $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}{testCase.Path}"); AvroReader avroReader = new AvroReader(stream); // Act object o = await avroReader.Next(async : true).ConfigureAwait(false); testCase.Predicate(o); } }
public virtual async Task <BlobChangeFeedEvent> Next( bool async, CancellationToken cancellationToken = default) { Dictionary <string, object> result; if (!HasNext()) { return(null); } result = (Dictionary <string, object>) await _avroReader.Next(async, cancellationToken).ConfigureAwait(false); BlockOffset = _avroReader.BlockOffset; EventIndex = _avroReader.ObjectIndex; return(new BlobChangeFeedEvent(result)); }
// Note - offset is with respect to buffer. private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count) { ValidateReadParameters(buffer, offset, count); int remainingBytes = _bufferLength - _bufferOffset; // We have enough bytes in the buffer and don't need to read the next Record. if (count <= remainingBytes) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: count); _bufferOffset += count; return(count); } // Copy remaining buffer if (remainingBytes > 0) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: remainingBytes); _bufferOffset += remainingBytes; return(remainingBytes); } // Reset _bufferOffset, _bufferLength, and remainingBytes _bufferOffset = 0; _bufferLength = 0; remainingBytes = 0; // We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream. if (!_avroReader.HasNext()) { return(0); } // We need to keep getting the next record until we get a data record. while (remainingBytes == 0) { // Get next Record. Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false); switch (record["$schema"]) { // Data Record case Constants.QuickQuery.DataRecordName: record.TryGetValue(Constants.QuickQuery.Data, out object byteObject); if (byteObject == null) { throw new InvalidOperationException($"Avro data record is missing {Constants.QuickQuery.Data} property"); } byte[] bytes = (byte[])byteObject; // Return the buffer if it is not null and not big enough. if (_buffer != null && _buffer.Length < bytes.Length) { ArrayPool <byte> .Shared.Return(_buffer, clearArray : true); } // Rent a new buffer if it is null or not big enough. if (_buffer == null || _buffer.Length < bytes.Length) { _buffer = ArrayPool <byte> .Shared.Rent(Math.Max(4 * Constants.MB, bytes.Length)); } Array.Copy( sourceArray: bytes, sourceIndex: 0, destinationArray: _buffer, destinationIndex: 0, length: bytes.Length); _bufferLength = bytes.Length; // Don't remove this reset, it is used in the final array copy below. remainingBytes = bytes.Length; break; // Progress Record case Constants.QuickQuery.ProgressRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress); if (progress == null) { throw new InvalidOperationException($"Avro progress record is mssing {Constants.QuickQuery.BytesScanned} property"); } _progressHandler.Report((long)progress); } break; // Error Record case Constants.QuickQuery.ErrorRecordName: ProcessErrorRecord(record); break; // End Record case Constants.QuickQuery.EndRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress); if (progress == null) { throw new InvalidOperationException($"Avro end record is missing {Constants.QuickQuery.TotalBytes} property"); } _progressHandler.Report((long)progress); } return(0); } } int length = Math.Min(count, remainingBytes); Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: length); _bufferOffset += length; return(length); }
// Note - offset is with respect to buffer. private async Task <int> ReadInternal(bool async, byte[] buffer, int offset, int count) { ValidateReadParameters(buffer, offset, count); int remainingBytes = _bufferLength - _bufferOffset; // We have enough bytes in the buffer and don't need to read the next Record. if (count <= remainingBytes) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: count); _bufferOffset += count; return(count); } // Copy remaining buffer if (remainingBytes > 0) { Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: remainingBytes); _bufferOffset += remainingBytes; return(remainingBytes); } // Reset _bufferOffset, _bufferLength, and remainingBytes _bufferOffset = 0; _bufferLength = 0; remainingBytes = 0; // We've caught up to the end of the _avroStream, but it isn't necessarly the end of the stream. // TODO what to do in this case? If we return 0, we are indicating the end of stream if (!_avroReader.HasNext()) { return(0); } // We need to keep getting the next record until we get a data record. while (remainingBytes == 0) { // Get next Record. Dictionary <string, object> record = (Dictionary <string, object>) await _avroReader.Next(async).ConfigureAwait(false); switch (record["$schema"]) { // Data Record case Constants.QuickQuery.DataRecordName: record.TryGetValue(Constants.QuickQuery.Data, out object byteObject); byte[] bytes = (byte[])byteObject; Array.Copy( sourceArray: bytes, sourceIndex: 0, destinationArray: _buffer, destinationIndex: 0, length: bytes.Length); _bufferLength = bytes.Length; // Don't remove this reset, it is used in the final array copy below. remainingBytes = bytes.Length; break; // Progress Record case Constants.QuickQuery.ProgressRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.BytesScanned, out object progress); _progressHandler.Report((long)progress); } break; // Error Record case Constants.QuickQuery.ErrorRecordName: ProcessErrorRecord(record); break; // End Record case Constants.QuickQuery.EndRecordName: if (_progressHandler != default) { record.TryGetValue(Constants.QuickQuery.TotalBytes, out object progress); _progressHandler.Report((long)progress); } return(0); } } int length = Math.Min(count, remainingBytes); Array.Copy( sourceArray: _buffer, sourceIndex: _bufferOffset, destinationArray: buffer, destinationIndex: offset, length: length); _bufferOffset += length; return(length); }