public void TestDifferentReaderSchema() { RecordSchema writerSchema = Schema.Parse("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}," + "{\"name\":\"f2\", \"type\":\"string\"}]}") as RecordSchema; Schema readerSchema = Schema.Parse("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"}," + "{\"name\":\"f3\", \"type\":\"string\", \"default\":\"test\"}]}"); MemoryStream dataFileOutputStream = new MemoryStream(); WriteGeneric(dataFileOutputStream, writerSchema, mkRecord(new [] { "f1", "f1val", "f2", "f2val" }, writerSchema), Codec.Type.Null); MemoryStream dataFileInputStream = new MemoryStream(dataFileOutputStream.ToArray()); using (IFileReader <GenericRecord> reader = DataFileReader <GenericRecord> .OpenReader(dataFileInputStream, readerSchema)) { GenericRecord result = reader.Next(); object ignore; Assert.IsFalse(result.TryGetValue("f2", out ignore)); Assert.AreEqual("f1val", result["f1"]); Assert.AreEqual("test", result["f3"]); } }
public KafkaConsumer(Dictionary <string, object> config, Action <string, dynamic, DateTime> consumeResultHandler, Action <string> logger) { if (consumeResultHandler == null || logger == null) { throw new Exception("Empty handler"); } _consumeResultHandler = consumeResultHandler; _logger = logger; _config = config; _cts = new CancellationTokenSource(); _genericRecordConfig = new RecordConfig((string)config[KafkaPropNames.SchemaRegistryUrl]); RecordSchema = _genericRecordConfig.RecordSchema; _topic = (string)config[KafkaPropNames.Topic]; _taskConsumer = StartConsumingInner(); }
public static GenericRecord DecodeRecord( this RecordSchema schema, JToken value) { var jvalue = value as JObject; if (jvalue != null) { var record = new GenericRecord(schema); foreach (var field in schema.Fields) { var property = jvalue.Property(field.Name); var rvalue = DecodeAny(field.Schema, property.Value); record.Put(field.Name, rvalue); } return(record); } throw new ArgumentException("invalid value type: " + value.GetType().FullName); }
public void RecordField(RecordSchema record, string expectedName, string expectedDoc, string[] expectedAliases) { var assembly = CodeGen.Compile(Guid.NewGuid().ToString(), record, out var xmlDocument); var type = assembly.ExportedTypes.FirstOrDefault(); Assert.NotNull(type); Assert.IsTrue(type.IsClass); Assert.IsTrue(typeof(IAvroRecord).IsAssignableFrom(type)); var property = type.GetProperty(expectedName); Assert.NotNull(property); var doc = GetSummaryText(xmlDocument, type.FullName, property.Name); Assert.AreEqual(expectedDoc, doc); var aliases = GetAliasList(xmlDocument, type.FullName, property.Name); Assert.AreEqual(expectedAliases, aliases); }
/// <summary> /// Generates a compilation unit (essentially a single .cs file) that contains types that /// match the schema. /// </summary> /// <param name="schema"> /// The schema to generate code for. Code can only be generated for enum /// and record schemas. /// </param> /// <returns> /// A compilation unit containing types that match the schema. /// </returns> /// <throws cref="UnsupportedSchemaException"> /// Thrown when the schema is not an enum or record, or when a record /// field schema is not recognized. /// </throws> public virtual CompilationUnitSyntax GenerateCompilationUnit(Schema schema) { var candidates = GetCandidateSchemas(schema) .OrderBy(s => s.Name) .GroupBy(s => s.Namespace) .OrderBy(g => g.Key); if (candidates.Count() < 1) { throw new UnsupportedSchemaException(schema, $"Code can only be generated for enums and records."); } var unit = SyntaxFactory.CompilationUnit(); foreach (var group in candidates) { var members = group .Select(candidate => candidate switch { EnumSchema enumSchema => GenerateEnum(enumSchema) as MemberDeclarationSyntax, RecordSchema recordSchema => GenerateClass(recordSchema) as MemberDeclarationSyntax, _ => default,
/// <summary> /// Generates a class declaration for a record schema. /// </summary> /// <param name="schema"> /// The schema to generate a class for. /// </param> /// <returns> /// A class declaration with a property for each field of the record schema. /// </returns> /// <throws cref="UnsupportedSchemaException"> /// Thrown when a field schema is not recognized. /// </throws> public virtual ClassDeclarationSyntax GenerateClass(RecordSchema schema) { var declaration = SyntaxFactory.ClassDeclaration(schema.Name) .AddModifiers(SyntaxFactory.Token(SyntaxKind.PublicKeyword)) .AddMembers(schema.Fields .Select(field => { var child = SyntaxFactory .PropertyDeclaration( GetPropertyType(field.Type), field.Name ) .AddModifiers(SyntaxFactory.Token(SyntaxKind.PublicKeyword)) .AddAccessorListAccessors( SyntaxFactory.AccessorDeclaration(SyntaxKind.GetAccessorDeclaration) .WithSemicolonToken(SyntaxFactory.Token(SyntaxKind.SemicolonToken)), SyntaxFactory.AccessorDeclaration(SyntaxKind.SetAccessorDeclaration) .WithSemicolonToken(SyntaxFactory.Token(SyntaxKind.SemicolonToken)) ); if (!string.IsNullOrEmpty(field.Documentation)) { child = AddSummaryComment(child, field.Documentation); } return(child); }) .Where(field => field != null) .ToArray() ); if (!string.IsNullOrEmpty(schema.Documentation)) { declaration = AddSummaryComment(declaration, schema.Documentation); } return(declaration); }
/// <summary> /// Event handler that fires any time there is a change in a OPC monitored variable /// </summary> /// <param name="emitter"></param> /// <param name="items"></param> public void OnNotification(object emitter, MonItemNotificationArgs items) { RecordSchema schema = schemas.GetSchema(items.dataType); if (schema != null) { foreach (var itm in items.values) { if (DataValue.IsBad(itm)) { continue; } var m = buildKafkaMessage(itm, schema, items.dataType, items.name); if (m == null) { continue; } // not waiting here var status = sendMessage(_conf.opcSystemName, m); log.Debug("Sending message {0}:{1} t:{2}", m.Key, m.Value, m.Timestamp.ToString()); } } }
internal Encoder.WriteItem Resolve(RecordSchema recordSchema) { WriteStep[] writeSteps = new WriteStep[recordSchema.Fields.Count]; int index = 0; foreach (Field field in recordSchema) { var record = new WriteStep { WriteField = Resolver.ResolveWriter(field.Schema), Field = field }; writeSteps[index++] = record; } void RecordResolver(object v, IWriter e) { WriteRecordFields(v, writeSteps, e); } return(RecordResolver); }
/// <summary> /// Ctor. /// </summary> /// <param name="eventType">type to create</param> /// <param name="eventAdapterService">event factory</param> /// <param name="properties">written properties</param> public EventBeanManufacturerAvro( AvroSchemaEventType eventType, EventAdapterService eventAdapterService, IList<WriteablePropertyDescriptor> properties) { _eventAdapterService = eventAdapterService; _eventType = (AvroEventType) eventType; _schema = _eventType.SchemaAvro; _indexPerWritable = new Field[properties.Count]; for (int i = 0; i < properties.Count; i++) { string propertyName = properties[i].PropertyName; var field = _schema.GetField(propertyName); if (field == null) { throw new IllegalStateException( "Failed to find property '" + propertyName + "' among the array indexes"); } _indexPerWritable[i] = field; } }
protected virtual IDictionary <string, object> ReadRecord(RecordSchema writerSchema, Schema readerSchema, IReader dec) { RecordSchema rs = (RecordSchema)readerSchema; Record result = new Record(rs); foreach (Field wf in writerSchema) { Field rf; if (rs.TryGetFieldAlias(wf.Name, out rf)) { object obj = null; TryGetField(result, wf.Name, rf.Pos, out obj); AddField(result, rf.aliases?[0] ?? wf.Name, rf.Pos, Read(wf.Schema, rf.Schema, dec)); } else { Skip(wf.Schema, dec); } } return(result.Contents); }
public void TestPrimitiveWithMetadata(string rawSchema, Schema.Type type) { Schema definedSchema = Schema.Parse(rawSchema); Assert.IsTrue(definedSchema is PrimitiveSchema); Assert.AreEqual(type.ToString().ToLower(), definedSchema.Name); Assert.AreEqual(type, definedSchema.Tag); testEquality(rawSchema, definedSchema); testToString(definedSchema); Assert.True(definedSchema.ToString().Contains("metafield")); var rawRecordSchema = "{\"type\":\"record\",\"name\":\"Foo\"," + "\"fields\":[{\"name\":\"f1\",\"type\":" + rawSchema + "}]}"; Schema baseRecordSchema = Schema.Parse(rawRecordSchema); Assert.AreEqual(Schema.Type.Record, baseRecordSchema.Tag); RecordSchema recordSchema = baseRecordSchema as RecordSchema; Assert.IsNotNull(recordSchema); Assert.AreEqual(1, recordSchema.Count); Assert.IsTrue(recordSchema["f1"].Schema is PrimitiveSchema); Assert.AreEqual(type.ToString().ToLower(), recordSchema["f1"].Schema.Name); Assert.AreEqual(type, recordSchema["f1"].Schema.Tag); testEquality(rawRecordSchema, baseRecordSchema); testToString(recordSchema["f1"].Schema); Assert.True(baseRecordSchema.ToString().Contains("metafield")); Assert.True(recordSchema["f1"].Schema.ToString().Contains("metafield")); Assert.True(definedSchema.Equals(recordSchema["f1"].Schema)); Assert.AreEqual(definedSchema.GetHashCode(), recordSchema["f1"].Schema.GetHashCode()); }
public void TestIt() { // Bean RunAssertion( "Bean", SupportEventInfra.FBEANWTYPE, new Bean_Type_Root(), new Bean_Type_1(), new Bean_Type_2(), new Bean_Type_2_1()); // Map RunAssertion( "Map", SupportEventInfra.FMAPWTYPE, new Dictionary <string, object>(), new Dictionary <string, object>(), new Dictionary <string, object>(), new Dictionary <string, object>()); // OA RunAssertion( "OA", SupportEventInfra.FOAWTYPE, new object[0], new object[0], new object[0], new object[0]); // Avro RecordSchema fake = SchemaBuilder.Record("fake"); RunAssertion( "Avro", SupportEventInfra.FAVROWTYPE, new GenericRecord(fake), new GenericRecord(fake), new GenericRecord(fake), new GenericRecord(fake)); }
public void RecordWithParallelDependencies() { var node = new RecordSchema("Node"); node.Fields.Add(new RecordField("Value", new IntSchema())); node.Fields.Add(new RecordField("Children", new ArraySchema(node))); var schema = new RecordSchema("Reference"); schema.Fields.Add(new RecordField("Node", node)); schema.Fields.Add(new RecordField("RelatedNodes", new ArraySchema(node))); var deserialize = deserializerBuilder.BuildDelegate <Reference>(schema); var serialize = serializerBuilder.BuildDelegate <Reference>(schema); using (stream) { serialize( new Reference() { Node = new Node() { Children = Array.Empty <Node>(), }, RelatedNodes = Array.Empty <Node>(), }, new Utf8JsonWriter(stream)); } var reader = new Utf8JsonReader(stream.ToArray()); var root = deserialize(ref reader); Assert.Empty(root.Node.Children); Assert.Empty(root.RelatedNodes); }
private static GenericRecord mkRecord(object[] kv, RecordSchema s) { GenericRecord input = new GenericRecord(s); for (int i = 0; i < kv.Length; i += 2) { string fieldName = (string)kv[i]; object fieldValue = kv[i + 1]; Schema inner = s[fieldName].Schema; if (inner is EnumSchema) { GenericEnum ge = new GenericEnum(inner as EnumSchema, (string)fieldValue); fieldValue = ge; } else if (inner is FixedSchema) { GenericFixed gf = new GenericFixed(inner as FixedSchema); gf.Value = (byte[])fieldValue; fieldValue = gf; } input.Add(fieldName, fieldValue); } return(input); }
private CsvToAvroGenericWriter(RecordSchema schema, string outputFilePath, Mode mode) { _avroSchema = schema; BuildDataFileWriter(outputFilePath, mode); }
/// <summary> /// Gets an instance of CsvToAvroGenericWriter. /// </summary> /// <param name="schema">An Avro RecordSchema object containing the schema.</param> /// <param name="outputFilePath">The path the Avro file should be written to.</param> /// <param name="mode"> /// If the output Avro file already exists, specifies whether it should be overwritten or appended to. /// The default is Overwrite. /// </param> /// <returns>A CsvToAvroGenericWriter object.</returns> public static CsvToAvroGenericWriter CreateFromSchema(RecordSchema schema, string outputFilePath, Mode mode = Mode.Overwrite) { return(new CsvToAvroGenericWriter(schema, outputFilePath, mode)); }
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers) { var s = (RecordSchema)RecordSchema.Parse( @"{ ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"", ""type"": ""record"", ""name"": ""User"", ""fields"": [ {""name"": ""name"", ""type"": ""string""}, {""name"": ""favorite_number"", ""type"": [""int"", ""null""]}, {""name"": ""favorite_color"", ""type"": [""string"", ""null""]} ] }" ); var config = new ProducerConfig { BootstrapServers = bootstrapServers }; var schemaRegistryConfig = new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryServers }; var topic = Guid.NewGuid().ToString(); DeliveryResult <Null, GenericRecord> dr; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var p = new ProducerBuilder <Null, GenericRecord>(config) .SetKeySerializer(Serializers.Null) .SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry)) .Build()) { var record = new GenericRecord(s); record.Add("name", "my name 2"); record.Add("favorite_number", 44); record.Add("favorite_color", null); dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> { Value = record }).Result; } // produce a specific record (to later consume back as a generic record). using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var p = new ProducerBuilder <Null, User>(config) .SetKeySerializer(Serializers.Null) .SetValueSerializer(new AvroSerializer <User>(schemaRegistry)) .Build()) { var user = new User { name = "my name 3", favorite_number = 47, favorite_color = "orange" }; p.ProduceAsync(topic, new Message <Null, User> { Value = user }).Wait(); } Assert.Null(dr.Message.Key); Assert.NotNull(dr.Message.Value); dr.Message.Value.TryGetValue("name", out object name); dr.Message.Value.TryGetValue("favorite_number", out object number); dr.Message.Value.TryGetValue("favorite_color", out object color); Assert.IsType <string>(name); Assert.IsType <int>(number); Assert.Equal("my name 2", name); Assert.Equal(44, number); Assert.Null(color); var cconfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers }; using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <Null, GenericRecord>(cconfig) .SetKeyDeserializer(Deserializers.Null) .SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync()) .Build()) { // consume generic record produced as a generic record. consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, dr.Offset) }); var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token); record.Message.Value.TryGetValue("name", out object msgName); record.Message.Value.TryGetValue("favorite_number", out object msgNumber); record.Message.Value.TryGetValue("favorite_color", out object msgColor); Assert.IsType <string>(msgName); Assert.IsType <int>(msgNumber); Assert.Equal("my name 2", msgName); Assert.Equal(44, msgNumber); Assert.Null(msgColor); // consume generic record produced as a specific record. record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token); record.Message.Value.TryGetValue("name", out msgName); record.Message.Value.TryGetValue("favorite_number", out msgNumber); record.Message.Value.TryGetValue("favorite_color", out msgColor); Assert.IsType <string>(msgName); Assert.IsType <int>(msgNumber); Assert.IsType <string>(msgColor); Assert.Equal("my name 3", msgName); Assert.Equal(47, msgNumber); Assert.Equal("orange", msgColor); } using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig)) using (var consumer = new ConsumerBuilder <Null, User>(cconfig) .SetKeyDeserializer(Deserializers.Null) .SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync()) .Build()) { consumer.Assign(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic, 0, dr.Offset) }); var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token); Assert.Equal("my name 2", record.Message.Value.name); Assert.Equal(44, record.Message.Value.favorite_number); Assert.Null(record.Message.Value.favorite_color); } }
public override void WriteRequest(RecordSchema schema, object request, Encoder encoder) { new GenericWriter<object>(schema).Write(request, encoder); }
public IEnumerator GetEnumerator() { var record01 = new RecordSchema() { Name = "TestRecord01" }; var record02 = new RecordSchema() { Name = "TestRecord02" }; var errord01 = new ErrorSchema() { Name = "TestError01" }; var errord02 = new ErrorSchema() { Name = "TestError02" }; var protocol01 = new AvroProtocol { Name = "ProtocolName" }; yield return(new object[] { protocol01, @"{""protocol"":""ProtocolName""}", @"{ ""protocol"": ""ProtocolName"" }", @"{ ""namespace"": """", ""protocol"": ""ProtocolName"", ""doc"": """", ""types"": [], ""messages"": {} }" }); var protocol02 = new AvroProtocol { Name = "ProtocolName", Namespace = "Test.Namespace" }; yield return(new object[] { protocol02, @"{""protocol"":""Test.Namespace.ProtocolName""}", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"" }", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": """", ""types"": [], ""messages"": {} }" }); var protocol03 = new AvroProtocol { Name = "ProtocolName", Namespace = "Test.Namespace", Doc = "Test Documentation" }; yield return(new object[] { protocol03, @"{""protocol"":""Test.Namespace.ProtocolName""}", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"" }", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": [], ""messages"": {} }" }); var protocol04 = new AvroProtocol { Name = "ProtocolName", Namespace = "Test.Namespace", Doc = "Test Documentation" }; protocol04.AddType(record01); protocol04.AddType(record02); protocol04.AddType(errord01); protocol04.AddType(errord02); yield return(new object[] { protocol04, @"{""protocol"":""Test.Namespace.ProtocolName"",""types"":" + @"[{""name"":""Test.Namespace.TestRecord01"",""type"":""record"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestRecord02"",""type"":""record"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestError01"",""type"":""error"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestError02"",""type"":""error"",""fields"":[]}" + @"]}", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " + @"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""fields"": [] }, " + @"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError01"", ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError02"", ""fields"": [] }" + @"] }", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " + @"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] " + @"}]," + @" ""messages"": {} }" }); var protocol05 = new AvroProtocol { Name = "ProtocolName", Namespace = "Test.Namespace", Doc = "Test Documentation" }; protocol05.AddType(record01); protocol05.AddType(record02); protocol05.AddType(errord01); protocol05.AddType(errord02); var message01 = new MessageSchema("M01") { Doc = "Test Doc 01" }; message01.AddParameter(new ParameterSchema("p01", record01)); message01.AddParameter(new ParameterSchema("p02", record02)); message01.AddError(errord01); message01.AddError(errord02); message01.Response = new DoubleSchema(); var message02 = new MessageSchema("M02"); message02.AddParameter(new ParameterSchema("p01", record01)); message02.AddError(errord01); message02.Oneway = true; protocol05.AddMessage(message01); protocol05.AddMessage(message02); yield return(new object[] { protocol05, @"{""protocol"":""Test.Namespace.ProtocolName"",""types"":" + @"[{""name"":""Test.Namespace.TestRecord01"",""type"":""record"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestRecord02"",""type"":""record"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestError01"",""type"":""error"",""fields"":[]}," + @"{""name"":""Test.Namespace.TestError02"",""type"":""error"",""fields"":[]}" + @"]," + @"""messages"":{" + @"""M01"":{" + @"""request"":[{""name"":""p01"",""type"":""Test.Namespace.TestRecord01""},{""name"":""p02"",""type"":""Test.Namespace.TestRecord02""}]," + @"""response"":""double""," + @"""errors"":[""Test.Namespace.TestError01"",""Test.Namespace.TestError02""]" + @"}," + @"""M02"":{" + @"""request"":[{""name"":""p01"",""type"":""Test.Namespace.TestRecord01""}]," + @"""errors"":[""Test.Namespace.TestError01""]," + @"""one-way"":true" + @"}}}", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " + @"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""fields"": [] }, " + @"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError01"", ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError02"", ""fields"": [] }" + @"], " + @"""messages"": {" + @"""M01"": {" + @"""doc"": ""Test Doc 01"", " + @"""request"": [{ ""name"": ""p01"", ""type"": ""TestRecord01"" }, { ""name"": ""p02"", ""type"": ""TestRecord02"" }], " + @"""response"": ""double"", " + @"""errors"": [""TestError01"", ""TestError02""]" + @"}, " + @"""M02"": {" + @"""request"": [{ ""name"": ""p01"", ""type"": ""TestRecord01"" }], " + @"""errors"": [""TestError01""], " + @"""one-way"": true" + @"}} }", @"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " + @"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " + @"{ ""type"": ""error"", ""name"": ""TestError02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }" + @"], " + @"""messages"": {" + @"""M01"": {" + @"""doc"": ""Test Doc 01"", " + @"""request"": [{ ""name"": ""p01"", ""type"": ""Test.Namespace.TestRecord01"" }, { ""name"": ""p02"", ""type"": ""Test.Namespace.TestRecord02"" }], " + @"""response"": { ""type"": ""double"" }, " + @"""errors"": [""Test.Namespace.TestError01"", ""Test.Namespace.TestError02""], " + @"""one-way"": false" + @"}, " + @"""M02"": {" + @"""doc"": """", " + @"""request"": [{ ""name"": ""p01"", ""type"": ""Test.Namespace.TestRecord01"" }], " + @"""response"": null, " + @"""errors"": [""Test.Namespace.TestError01""], " + @"""one-way"": true" + @"}} }" }); }
static async Task Main(string[] args) { if (args.Length != 3) { Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName"); return; } string bootstrapServers = args[0]; string schemaRegistryUrl = args[1]; string topicName = args[2]; string groupName = "avro-generic-example-group"; // var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json")); var s = (RecordSchema)RecordSchema.Parse( @"{ ""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"", ""type"": ""record"", ""name"": ""User"", ""fields"": [ {""name"": ""name"", ""type"": ""string""}, {""name"": ""favorite_number"", ""type"": [""int"", ""null""]}, {""name"": ""favorite_color"", ""type"": [""string"", ""null""]} ] }" ); CancellationTokenSource cts = new CancellationTokenSource(); var consumeTask = Task.Run(() => { using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl })) using (var consumer = new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = groupName }) .SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry)) .SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry)) .SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}")) .Build()) { consumer.Subscribe(topicName); try { while (true) { try { var consumeResult = consumer.Consume(cts.Token); Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}"); } catch (ConsumeException e) { Console.WriteLine($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // commit final offsets and leave the group. consumer.Close(); } } }); using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig { SchemaRegistryUrl = schemaRegistryUrl })) using (var producer = new ProducerBuilder <string, GenericRecord>(new ProducerConfig { BootstrapServers = bootstrapServers }) .SetKeySerializer(new AvroSerializer <string>(schemaRegistry)) .SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry)) .Build()) { Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit."); int i = 0; string text; while ((text = Console.ReadLine()) != "q") { var record = new GenericRecord(s); record.Add("name", text); record.Add("favorite_number", i++); record.Add("favorite_color", "blue"); await producer .ProduceAsync(topicName, new Message <string, GenericRecord> { Key = text, Value = record }) .ContinueWith(task => task.IsFaulted ? $"error producing message: {task.Exception.Message}" : $"produced to: {task.Result.TopicPartitionOffset}"); } } cts.Cancel(); }
public bool SchemaMatches(RecordSchema schema) { return(_schema.Equals(schema)); }
/// <summary> /// Reads the passed JToken default value field and writes it in the specified encoder /// </summary> /// <param name="enc">encoder to use for writing</param> /// <param name="schema">schema object for the current field</param> /// <param name="jtok">default value as JToken</param> public static void EncodeDefaultValue(Encoder enc, Schema schema, JToken jtok) { if (null == jtok) { return; } switch (schema.Tag) { case Schema.Type.Boolean: if (jtok.Type != JTokenType.Boolean) { throw new AvroException("Default boolean value " + jtok.ToString() + " is invalid, expected is json boolean."); } enc.WriteBoolean((bool)jtok); break; case Schema.Type.Int: if (jtok.Type != JTokenType.Integer) { throw new AvroException("Default int value " + jtok.ToString() + " is invalid, expected is json integer."); } enc.WriteInt(Convert.ToInt32((int)jtok)); break; case Schema.Type.Long: if (jtok.Type != JTokenType.Integer) { throw new AvroException("Default long value " + jtok.ToString() + " is invalid, expected is json integer."); } enc.WriteLong(Convert.ToInt64((long)jtok)); break; case Schema.Type.Float: if (jtok.Type != JTokenType.Float) { throw new AvroException("Default float value " + jtok.ToString() + " is invalid, expected is json number."); } enc.WriteFloat((float)jtok); break; case Schema.Type.Double: if (jtok.Type == JTokenType.Integer) { enc.WriteDouble(Convert.ToDouble((int)jtok)); } else if (jtok.Type == JTokenType.Float) { enc.WriteDouble(Convert.ToDouble((float)jtok)); } else { throw new AvroException("Default double value " + jtok.ToString() + " is invalid, expected is json number."); } break; case Schema.Type.Bytes: if (jtok.Type != JTokenType.String) { throw new AvroException("Default bytes value " + jtok.ToString() + " is invalid, expected is json string."); } var en = System.Text.Encoding.GetEncoding("iso-8859-1"); enc.WriteBytes(en.GetBytes((string)jtok)); break; case Schema.Type.Fixed: if (jtok.Type != JTokenType.String) { throw new AvroException("Default fixed value " + jtok.ToString() + " is invalid, expected is json string."); } en = System.Text.Encoding.GetEncoding("iso-8859-1"); int len = (schema as FixedSchema).Size; byte[] bb = en.GetBytes((string)jtok); if (bb.Length != len) { throw new AvroException("Default fixed value " + jtok.ToString() + " is not of expected length " + len); } enc.WriteFixed(bb); break; case Schema.Type.String: if (jtok.Type != JTokenType.String) { throw new AvroException("Default string value " + jtok.ToString() + " is invalid, expected is json string."); } enc.WriteString((string)jtok); break; case Schema.Type.Enumeration: if (jtok.Type != JTokenType.String) { throw new AvroException("Default enum value " + jtok.ToString() + " is invalid, expected is json string."); } enc.WriteEnum((schema as EnumSchema).Ordinal((string)jtok)); break; case Schema.Type.Null: if (jtok.Type != JTokenType.Null) { throw new AvroException("Default null value " + jtok.ToString() + " is invalid, expected is json null."); } enc.WriteNull(); break; case Schema.Type.Array: if (jtok.Type != JTokenType.Array) { throw new AvroException("Default array value " + jtok.ToString() + " is invalid, expected is json array."); } JArray jarr = jtok as JArray; enc.WriteArrayStart(); enc.SetItemCount(jarr.Count); foreach (JToken jitem in jarr) { enc.StartItem(); EncodeDefaultValue(enc, (schema as ArraySchema).ItemSchema, jitem); } enc.WriteArrayEnd(); break; case Schema.Type.Record: case Schema.Type.Error: if (jtok.Type != JTokenType.Object) { throw new AvroException("Default record value " + jtok.ToString() + " is invalid, expected is json object."); } RecordSchema rcs = schema as RecordSchema; JObject jo = jtok as JObject; foreach (Field field in rcs) { JToken val = jo[field.Name]; if (null == val) { val = field.DefaultValue; } if (null == val) { throw new AvroException("No default value for field " + field.Name); } EncodeDefaultValue(enc, field.Schema, val); } break; case Schema.Type.Map: if (jtok.Type != JTokenType.Object) { throw new AvroException("Default map value " + jtok.ToString() + " is invalid, expected is json object."); } jo = jtok as JObject; enc.WriteMapStart(); enc.SetItemCount(jo.Count); foreach (KeyValuePair <string, JToken> jp in jo) { enc.StartItem(); enc.WriteString(jp.Key); EncodeDefaultValue(enc, (schema as MapSchema).ValueSchema, jp.Value); } enc.WriteMapEnd(); break; case Schema.Type.Union: enc.WriteUnionIndex(0); EncodeDefaultValue(enc, (schema as UnionSchema).Schemas[0], jtok); break; default: throw new AvroException("Unsupported schema type " + schema.Tag); } }
public void RecordWithMissingFields() { var boolean = new BooleanSchema(); var array = new ArraySchema(boolean); var map = new MapSchema(new IntSchema()); var @enum = new EnumSchema("Ordinal", new[] { "None", "First", "Second", "Third", "Fourth" }); var union = new UnionSchema(new Schema[] { new NullSchema(), array, }); var schema = new RecordSchema("AllFields") { Fields = new[] { new RecordField("First", union), new RecordField("Second", union), new RecordField("Third", array), new RecordField("Fourth", array), new RecordField("Fifth", map), new RecordField("Sixth", map), new RecordField("Seventh", @enum), new RecordField("Eighth", @enum), }, }; var deserialize = deserializerBuilder.BuildDelegate <WithoutEvenFields>(schema); var serialize = serializerBuilder.BuildDelegate <WithEvenFields>(schema); var value = new WithEvenFields() { First = new List <bool>() { false }, Second = new List <bool>() { false, false }, Third = new List <bool>() { false, false, false }, Fourth = new List <bool>() { false }, Fifth = new Dictionary <string, int>() { { "first", 1 } }, Sixth = new Dictionary <string, int>() { { "first", 1 }, { "second", 2 } }, Seventh = ImplicitEnum.First, Eighth = ImplicitEnum.None, }; using (stream) { serialize(value, new Utf8JsonWriter(stream)); } var reader = new Utf8JsonReader(stream.ToArray()); Assert.Equal(value.Seventh, deserialize(ref reader).Seventh); }
protected override RecordAccess GetRecordAccess(RecordSchema schema) { return(new GenericRecordAccess(schema)); }
protected abstract void EnsureRecordObject(RecordSchema recordSchema, object value);
public GenericRecord(RecordSchema schema) { this.Schema = schema; }
protected override object ReadRecord(object reuse, RecordSchema writerSchema, Schema readerSchema, Decoder decoder) { reuse = reuse ?? CreateInstance(readerSchema, Schema.Type.Record); return(base.ReadRecord(reuse, writerSchema, readerSchema, decoder)); }
public SpecificRecordAccess(RecordSchema readerSchema) { objCreator = GetConstructor(readerSchema.Fullname, Schema.Type.Record); }
private ReadItem ResolveRecord(RecordSchema writerSchema, RecordSchema readerSchema) { var schemaPair = new SchemaPair(writerSchema, readerSchema); ReadItem recordReader; if (_recordReaders.TryGetValue(schemaPair, out recordReader)) { return(recordReader); } FieldReader[] fieldReaderArray = null; var recordAccess = GetRecordAccess(readerSchema); recordReader = (r, d) => ReadRecord(r, d, recordAccess, fieldReaderArray); _recordReaders.Add(schemaPair, recordReader); var readSteps = new List <FieldReader>(); foreach (Field wf in writerSchema) { Field rf; if (readerSchema.TryGetFieldAlias(wf.Name, out rf)) { var readItem = ResolveReader(wf.Schema, rf.Schema); if (IsReusable(rf.Schema.Tag)) { readSteps.Add((rec, d) => recordAccess.AddField(rec, rf.Name, rf.Pos, readItem(recordAccess.GetField(rec, rf.Name, rf.Pos), d))); } else { readSteps.Add((rec, d) => recordAccess.AddField(rec, rf.Name, rf.Pos, readItem(null, d))); } } else { var skip = GetSkip(wf.Schema); readSteps.Add((rec, d) => skip(d)); } } // fill in defaults for any reader fields not in the writer schema foreach (Field rf in readerSchema) { if (writerSchema.Contains(rf.Name)) { continue; } var defaultStream = new MemoryStream(); var defaultEncoder = new BinaryEncoder(defaultStream); defaultStream.Position = 0; // reset for writing Resolver.EncodeDefaultValue(defaultEncoder, rf.Schema, rf.DefaultValue); defaultStream.Flush(); var defaultBytes = defaultStream.ToArray(); var readItem = ResolveReader(rf.Schema, rf.Schema); var rfInstance = rf; if (IsReusable(rf.Schema.Tag)) { readSteps.Add((rec, d) => recordAccess.AddField(rec, rfInstance.Name, rfInstance.Pos, readItem(recordAccess.GetField(rec, rfInstance.Name, rfInstance.Pos), new BinaryDecoder(new MemoryStream(defaultBytes))))); } else { readSteps.Add((rec, d) => recordAccess.AddField(rec, rfInstance.Name, rfInstance.Pos, readItem(null, new BinaryDecoder(new MemoryStream(defaultBytes))))); } } fieldReaderArray = readSteps.ToArray(); return(recordReader); }
public GenericRecordAccess(RecordSchema schema) { _schema = schema; }
protected abstract RecordAccess GetRecordAccess(RecordSchema readerSchema);