public void TestFingerprint(string avro, long expectedFp) { var s = AvroParser.ReadSchema(avro); var fp = AvroFingerprint.CRC64Value(s); Assert.AreEqual(expectedFp, fp); }
public void SchemaParseBasic(Type expectedSchema, string avroString, string expectedToString) { var schema = AvroParser.ReadSchema(avroString); Assert.AreEqual(expectedSchema, schema.GetType()); Assert.AreEqual(expectedToString, schema.ToString()); }
private Header(Header header, FileInfo fileInfo) { _fileInfo = fileInfo; Magic = header.Magic.Clone() as byte[]; Sync = header.Sync.Clone() as byte[]; Codec = header.Codec; Schema = AvroParser.ReadSchema(header.Schema.ToAvro()); FileHeaderSize = header.FileHeaderSize; Metadata = new Dictionary <string, byte[]>(); foreach (var keyValue in header.Metadata) { var key = string.Copy(keyValue.Key); var value = keyValue.Value.Clone() as byte[]; Metadata.Add(key, value); } }
private void UpdateFromFile() { using (var stream = FileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.Read)) using (var decoder = new BinaryDecoder(stream)) { Magic = decoder.ReadFixed(4); if (!Magic.SequenceEqual(MAGIC_BYTES)) { throw new Exception("Invalid Avro file"); } Metadata = decoder.ReadMap(s => s.ReadBytes()); if (Metadata.TryGetValue("avro.schema", out var schemaBytes)) { Schema = AvroParser.ReadSchema(Encoding.UTF8.GetString(schemaBytes)); } else { throw new Exception("Schema not found"); } if (Metadata.TryGetValue("avro.codec", out var codecBytes)) { if (Enum.TryParse <Codec>(Encoding.UTF8.GetString(codecBytes), true, out var codec)) { Codec = codec; } else { throw new Exception("Codec not found"); } } else { Codec = null; } Sync = decoder.ReadFixed(16); FileHeaderSize = stream.Position; } }
public void RecordGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var fixedSchema = AvroParser.ReadSchema <FixedSchema>(@"{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}"); var subRecordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}"); var recordSchema = AvroParser.ReadSchema <RecordSchema>(@"{""name"":""Avro.Test.Generic.TestRecord"",""type"":""record"",""fields"":[{""name"":""FieldA"",""type"":""int""},{""name"":""FieldB"",""type"":""string""},{""name"":""FieldC"",""type"":{""name"":""Avro.Test.Generic.TestSubRecord"",""type"":""record"",""fields"":[{""name"":""FieldD"",""type"":""boolean""}]}},{""name"":""FieldX"",""type"":{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}},{""name"":""TestFixed"",""type"":{""name"":""Avro.Test.Generic.TestFixed"",""type"":""fixed"",""size"":40}}]}"); var subRecordInstance = new GenericRecord(subRecordSchema); var fixedInstance = new GenericFixed(fixedSchema); var enumInstance = new GenericEnum(enumSchema, "B"); subRecordInstance["FieldD"] = false; var expectedValue = new GenericRecord(recordSchema); expectedValue["FieldA"] = 123; expectedValue["FieldB"] = "Test"; expectedValue["FieldC"] = subRecordInstance; expectedValue["FieldX"] = enumInstance; expectedValue["TestFixed"] = fixedInstance; var writer = new DatumWriter <GenericRecord>(recordSchema); var reader = new DatumReader <GenericRecord>(recordSchema, recordSchema); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); for (int i = 0; i < expectedValue.FieldCount; i++) { Assert.AreEqual(expectedValue[i], actualValue[i]); } } }
public void EnumGenericTest() { var enumSchema = AvroParser.ReadSchema <EnumSchema>(@"{""name"":""Avro.Test.Generic.TestEnum"",""type"":""enum"",""symbols"":[""A"",""B"",""C""]}"); var writer = new DatumWriter <GenericEnum>(enumSchema); var reader = new DatumReader <GenericEnum>(enumSchema, enumSchema); var expectedValue = new GenericEnum(enumSchema, "B"); using (var stream = new MemoryStream()) using (var encoder = new BinaryEncoder(stream)) using (var decoder = new BinaryDecoder(stream)) { writer.Write(encoder, expectedValue); stream.Seek(0, SeekOrigin.Begin); var actualValue = reader.Read(decoder); Assert.AreEqual(expectedValue, actualValue); var expectedPosition = stream.Position; stream.Seek(0, SeekOrigin.Begin); reader.Skip(decoder); Assert.AreEqual(expectedPosition, stream.Position); } }
public void TestSimple() { var schema = AvroParser.ReadSchema(@"{ ""name"":""Test.Foobar.Record.Thing"", ""type"":""record"", ""fields"":[ {""name"":""ID"",""type"":""int""}, {""name"":""Name"",""type"":""string""}, {""name"":""Nuller"",""type"":""null""}, {""name"":""family"",""type"":{ ""type"":""map"",""values"":""string"" }}, {""name"":""stuff"",""type"":{ ""name"":""stuffs"", ""type"":""record"", ""fields"":[ {""name"":""key"",""type"":""bytes""}, {""name"":""keytype"",""type"":{ ""name"":""keyenum"", ""type"":""enum"", ""symbols"":[""ANALOG"",""DIGITAL""] }} ] }}, {""name"":""NullableThing"",""type"":[""null"",""string""]}, {""name"":""Tags"",""type"":{ ""type"":""array"",""items"":""string"" }} ] }") as RecordSchema; var stuffSchema = schema.First(r => r.Name == "stuff").Type as RecordSchema; var enumSchema = stuffSchema.First(r => r.Name == "keytype").Type as EnumSchema; var genericRecord = new GenericRecord(schema); var reader = new DatumReader <GenericRecord>(schema); var writer = new DatumWriter <GenericRecord>(schema); var delimiter = Environment.NewLine; var stringBuilder = new StringBuilder(); using (var stream = new StringWriter(stringBuilder)) using (var encoder = new JsonEncoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = new GenericRecord(genericRecord); record[0] = i; record[1] = $"foo{i}"; record[2] = null; record[3] = new Dictionary <string, string>() { { "Brother", "John" } }; var stuffRecord = new GenericRecord(stuffSchema); var keyEnum = new GenericEnum(enumSchema, i % enumSchema.Symbols.Count); stuffRecord[0] = Guid.NewGuid().ToByteArray(); stuffRecord[1] = keyEnum; record[4] = stuffRecord; record[5] = (i % 2) == 0 ? "ToNullOrNotToNull" : null; record[6] = new List <string> { }; writer.Write(encoder, record); var singleJson = stringBuilder.ToString(); } } var allJson = stringBuilder.ToString(); using (var stream = new StringReader(allJson)) using (var decoder = new JsonDecoder(stream, schema, delimiter)) { for (int i = 0; i < 10; i++) { var record = reader.Read(decoder); Debug.WriteLine(record.ToString()); } } }
public void SchemaParseException(string avroString, Type expectedExceptionType) { Assert.Throws(expectedExceptionType, () => { AvroParser.ReadSchema(avroString); }); }