public void TestFingerprint(string avro, long expectedFp)
{
var s = AvroParser.ReadSchema(avro);
var fp = AvroFingerprint.CRC64Value(s);
Assert.AreEqual(expectedFp, fp);
}
public void ProtocolWrite(AvroProtocol protocol, string expectedCanonicalAvro, string expectedDefaultAvro, string expectedFullAvro)
{
var canonicalAvro = new StringBuilder();
using (var writer = new StringWriter(canonicalAvro))
AvroParser.WriteAvroCanonical(writer, protocol);
var actualCanonicalAvro = canonicalAvro.ToString();
var defaultAvro = new StringBuilder();
using (var writer = new StringWriter(defaultAvro))
AvroParser.WriteAvro(writer, protocol);
var actualDefaultAvro = defaultAvro.ToString();
var fullAvro = new StringBuilder();
using (var writer = new StringWriter(fullAvro))
AvroParser.WriteAvroFull(writer, protocol);
var actualFullAvro = fullAvro.ToString();
Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Canonical form mismatch");
Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Default form mismatch");
Assert.AreEqual(expectedFullAvro, actualFullAvro, "Full form mismatch");
actualCanonicalAvro = protocol.ToAvroCanonical();
actualDefaultAvro = protocol.ToAvro();
actualFullAvro = protocol.ToAvroFull();
Assert.AreEqual(expectedCanonicalAvro, actualCanonicalAvro, "Extension - Canonical form mismatch");
Assert.AreEqual(expectedDefaultAvro, actualDefaultAvro, "Extension - Default form mismatch");
Assert.AreEqual(expectedFullAvro, actualFullAvro, "Extension - Full form mismatch");
}
public void SchemaParseBasic(Type expectedSchema, string avroString, string expectedToString)
{
var schema = AvroParser.ReadSchema(avroString);
Assert.AreEqual(expectedSchema, schema.GetType());
Assert.AreEqual(expectedToString, schema.ToString());
}
public void ProtocolParseException(string avroString, Type expectedExceptionType)
{
Assert.Throws(
expectedExceptionType,
() => AvroParser.ReadProtocol(avroString)
);
}
public void ProtocolParseBasic(string protocolAvro, string baseProtocolAvro)
{
var expectedMD5 = AvroParser.ReadProtocol(baseProtocolAvro).MD5;
var actualMD5 = AvroParser.ReadProtocol(protocolAvro).MD5;
Assert.AreEqual(expectedMD5, actualMD5);
}
public void UnknownSchemaTest()
{
var schema = new UnknownSchema();
var canonicalAvro = new StringBuilder();
using (var writer = new StringWriter(canonicalAvro))
Assert.Throws(typeof(AvroException), () => AvroParser.WriteAvroCanonical(writer, schema));
}
private Header(Header header, FileInfo fileInfo)
{
_fileInfo = fileInfo;
Magic = header.Magic.Clone() as byte[];
Sync = header.Sync.Clone() as byte[];
Codec = header.Codec;
Schema = AvroParser.ReadSchema(header.Schema.ToAvro());
FileHeaderSize = header.FileHeaderSize;
Metadata = new Dictionary <string, byte[]>();
foreach (var keyValue in header.Metadata)
{
var key = string.Copy(keyValue.Key);
var value = keyValue.Value.Clone() as byte[];
Metadata.Add(key, value);
}
}
private void UpdateFromFile()
{
using (var stream = FileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.Read))
using (var decoder = new BinaryDecoder(stream))
{
Magic = decoder.ReadFixed(4);
if (!Magic.SequenceEqual(MAGIC_BYTES))
{
throw new Exception("Invalid Avro file");
}
Metadata = decoder.ReadMap(s => s.ReadBytes());
if (Metadata.TryGetValue("avro.schema", out var schemaBytes))
{
Schema = AvroParser.ReadSchema(Encoding.UTF8.GetString(schemaBytes));
}
else
{
throw new Exception("Schema not found");
}
if (Metadata.TryGetValue("avro.codec", out var codecBytes))
{
if (Enum.TryParse <Codec>(Encoding.UTF8.GetString(codecBytes), true, out var codec))
{
Codec = codec;
}
else
{
throw new Exception("Codec not found");
}
}
else
{
Codec = null;
}
Sync = decoder.ReadFixed(16);
FileHeaderSize = stream.Position;
}
}
static async Task Main(string[] args)
{
// TODO: use something different for arg management.
var connectionString = args[0];
var containerName = args[1];
var fileName = args[2];
if (String.IsNullOrWhiteSpace(fileName))
{
Console.WriteLine("Usage: ConsoleSample.exe {connectionString} {containerName} {fileName}");
}
else
{
var items = await AvroParser.ParseDataFromCloudStorageAsync(connectionString, containerName, fileName);
Console.WriteLine("Parsed {0} items", items.Count);
foreach (var item in items)
{
Console.WriteLine("Enqueued Time: " + item.EnqueuedTimeUtc.ToLongDateString());
}
}
}
public void RecordGenericTest()
{
var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");
var fixedSchema = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}");
var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}");
var recordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}");
var subRecordInstance = new GenericRecord(subRecordSchema);
var fixedInstance = new GenericFixed(fixedSchema);
var enumInstance = new GenericEnum(enumSchema, "B");
subRecordInstance["FieldD"] = false;
var expectedValue = new GenericRecord(recordSchema);
expectedValue["FieldA"] = 123;
expectedValue["FieldB"] = "Test";
expectedValue["FieldC"] = subRecordInstance;
expectedValue["FieldX"] = enumInstance;
expectedValue["TestFixed"] = fixedInstance;
var writer = new DatumWriter <GenericRecord>(recordSchema);
var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema);
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
for (int i = 0; i < expectedValue.FieldCount; i++)
{
Assert.AreEqual(expectedValue[i], actualValue[i]);
}
}
}
public void EnumGenericTest()
{
var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}");
var writer = new DatumWriter <GenericEnum>(enumSchema);
var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema);
var expectedValue = new GenericEnum(enumSchema, "B");
using (var stream = new MemoryStream())
using (var encoder = new BinaryEncoder(stream))
using (var decoder = new BinaryDecoder(stream))
{
writer.Write(encoder, expectedValue);
stream.Seek(0, SeekOrigin.Begin);
var actualValue = reader.Read(decoder);
Assert.AreEqual(expectedValue, actualValue);
var expectedPosition = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
reader.Skip(decoder);
Assert.AreEqual(expectedPosition, stream.Position);
}
}
private async Task <GenericContext> Request(GenericContext rpcContext, CancellationToken token)
{
using (var requestData = new FrameStream())
using (var encoder = new BinaryEncoder(requestData))
{
if (DoHandshake())
{
if (rpcContext.HandshakeRequest == null)
{
rpcContext.HandshakeRequest = NewHandshakeRequest((MD5)Protocol.MD5, (MD5)RemoteProtocol.MD5);
}
HANDSHAKE_REQUEST_WRITER.Write(encoder, rpcContext.HandshakeRequest);
}
META_WRITER.Write(encoder, EMPTY_META);
encoder.WriteString(rpcContext.MessageName);
_protocol.WriteRequest(encoder, rpcContext.MessageName, rpcContext.RequestParameters);
encoder.WriteBytes(END_OF_FRAME);
requestData.Seek(0, SeekOrigin.Begin);
using (var responseData = await _tranceiver.RequestAsync(rpcContext.MessageName, requestData, token))
using (var decode = new BinaryDecoder(responseData))
{
responseData.Seek(0, SeekOrigin.Begin);
if (DoHandshake())
{
rpcContext.HandshakeResponse = HANDSHAKE_RESPONSE_READER.Read(decode);
_handshakePending = rpcContext.HandshakeResponse.match == HandshakeMatch.NONE;
var remoteProtocol = default(AvroProtocol);
if (rpcContext.HandshakeResponse.match == HandshakeMatch.CLIENT || rpcContext.HandshakeResponse.match == HandshakeMatch.NONE)
{
remoteProtocol = AvroParser.ReadProtocol(rpcContext.HandshakeResponse.serverProtocol);
_protocol = new GenericRequestor(Protocol, remoteProtocol);
}
if (rpcContext.HandshakeResponse.match == HandshakeMatch.NONE)
{
rpcContext.HandshakeRequest.serverHash = (MD5)remoteProtocol.MD5;
rpcContext.HandshakeRequest.clientProtocol = Protocol.ToAvroCanonical();
_protocol = new GenericRequestor(Protocol, remoteProtocol);
rpcContext = await Request(rpcContext, token);
}
}
rpcContext.Metadata = META_READER.Read(decode);
rpcContext.IsError = decode.ReadBoolean();
if (rpcContext.IsError)
{
rpcContext.Error = _protocol.ReadError <object>(decode, rpcContext.MessageName);
}
else
{
rpcContext.Response = _protocol.ReadResponse <object>(decode, rpcContext.MessageName);
}
return(rpcContext);
}
}
}
public void ProtocolParseBasic(string avroString, string expectedToString)
{
var protocol = AvroParser.ReadProtocol(avroString);
Assert.AreEqual(expectedToString, protocol.ToString());
}
public void TestSimple()
{
var schema = AvroParser.ReadSchema(@"{
""name"":""Test.Foobar.Record.Thing"",
""type"":""record"",
""fields"":[
{""name"":""ID"",""type"":""int""},
{""name"":""Name"",""type"":""string""},
{""name"":""Nuller"",""type"":""null""},
{""name"":""family"",""type"":{
""type"":""map"",""values"":""string""
}},
{""name"":""stuff"",""type"":{
""name"":""stuffs"",
""type"":""record"",
""fields"":[
{""name"":""key"",""type"":""bytes""},
{""name"":""keytype"",""type"":{
""name"":""keyenum"",
""type"":""enum"",
""symbols"":[""ANALOG"",""DIGITAL""]
}}
]
}},
{""name"":""NullableThing"",""type"":[""null"",""string""]},
{""name"":""Tags"",""type"":{
""type"":""array"",""items"":""string""
}}
]
}") as RecordSchema;
var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema;
var enumSchema = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema;
var genericRecord = new GenericRecord(schema);
var reader = new DatumReader <GenericRecord>(schema);
var writer = new DatumWriter <GenericRecord>(schema);
var delimiter = Environment.NewLine;
var stringBuilder = new StringBuilder();
using (var stream = new StringWriter(stringBuilder))
using (var encoder = new JsonEncoder(stream, schema, delimiter))
{
for (int i = 0; i < 10; i++)
{
var record = new GenericRecord(genericRecord);
record[0] = i;
record[1] = $"foo{i}";
record[2] = null;
record[3] = new Dictionary <string, string>()
{
{ "Brother", "John" }
};
var stuffRecord = new GenericRecord(stuffSchema);
var keyEnum = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count);
stuffRecord[0] = Guid.NewGuid().ToByteArray();
stuffRecord[1] = keyEnum;
record[4] = stuffRecord;
record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null;
record[6] = new List <string> {
};
writer.Write(encoder, record);
var singleJson = stringBuilder.ToString();
}
}
var allJson = stringBuilder.ToString();
using (var stream = new StringReader(allJson))
using (var decoder = new JsonDecoder(stream, schema, delimiter))
{
for (int i = 0; i < 10; i++)
{
var record = reader.Read(decoder);
Debug.WriteLine(record.ToString());
}
}
}
public void SchemaParseException(string avroString, Type expectedExceptionType)
{
Assert.Throws(expectedExceptionType, () => { AvroParser.ReadSchema(avroString); });
}
