public void TestDifferentReaderSchema()
{
RecordSchema writerSchema = Schema.Parse("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"},"
+ "{\"name\":\"f2\", \"type\":\"string\"}]}") as RecordSchema;
Schema readerSchema = Schema.Parse("{\"type\":\"record\", \"name\":\"n\", \"fields\":[{\"name\":\"f1\", \"type\":\"string\"},"
+ "{\"name\":\"f3\", \"type\":\"string\", \"default\":\"test\"}]}");
MemoryStream dataFileOutputStream = new MemoryStream();
WriteGeneric(dataFileOutputStream, writerSchema, mkRecord(new [] { "f1", "f1val", "f2", "f2val" }, writerSchema), Codec.Type.Null);
MemoryStream dataFileInputStream = new MemoryStream(dataFileOutputStream.ToArray());
using (IFileReader <GenericRecord> reader = DataFileReader <GenericRecord> .OpenReader(dataFileInputStream, readerSchema))
{
GenericRecord result = reader.Next();
object ignore;
Assert.IsFalse(result.TryGetValue("f2", out ignore));
Assert.AreEqual("f1val", result["f1"]);
Assert.AreEqual("test", result["f3"]);
}
}
public KafkaConsumer(Dictionary <string, object> config,
Action <string, dynamic, DateTime> consumeResultHandler,
Action <string> logger)
{
if (consumeResultHandler == null || logger == null)
{
throw new Exception("Empty handler");
}
_consumeResultHandler = consumeResultHandler;
_logger = logger;
_config = config;
_cts = new CancellationTokenSource();
_genericRecordConfig = new RecordConfig((string)config[KafkaPropNames.SchemaRegistryUrl]);
RecordSchema = _genericRecordConfig.RecordSchema;
_topic = (string)config[KafkaPropNames.Topic];
_taskConsumer = StartConsumingInner();
}
public static GenericRecord DecodeRecord(
this RecordSchema schema,
JToken value)
{
var jvalue = value as JObject;
if (jvalue != null)
{
var record = new GenericRecord(schema);
foreach (var field in schema.Fields)
{
var property = jvalue.Property(field.Name);
var rvalue = DecodeAny(field.Schema, property.Value);
record.Put(field.Name, rvalue);
}
return(record);
}
throw new ArgumentException("invalid value type: " + value.GetType().FullName);
}
public void RecordField(RecordSchema record, string expectedName, string expectedDoc, string[] expectedAliases)
{
var assembly = CodeGen.Compile(Guid.NewGuid().ToString(), record, out var xmlDocument);
var type = assembly.ExportedTypes.FirstOrDefault();
Assert.NotNull(type);
Assert.IsTrue(type.IsClass);
Assert.IsTrue(typeof(IAvroRecord).IsAssignableFrom(type));
var property = type.GetProperty(expectedName);
Assert.NotNull(property);
var doc = GetSummaryText(xmlDocument, type.FullName, property.Name);
Assert.AreEqual(expectedDoc, doc);
var aliases = GetAliasList(xmlDocument, type.FullName, property.Name);
Assert.AreEqual(expectedAliases, aliases);
}
/// <summary>
/// Generates a compilation unit (essentially a single .cs file) that contains types that
/// match the schema.
/// </summary>
/// <param name="schema">
/// The schema to generate code for. Code can only be generated for enum
/// and record schemas.
/// </param>
/// <returns>
/// A compilation unit containing types that match the schema.
/// </returns>
/// <throws cref="UnsupportedSchemaException">
/// Thrown when the schema is not an enum or record, or when a record
/// field schema is not recognized.
/// </throws>
public virtual CompilationUnitSyntax GenerateCompilationUnit(Schema schema)
{
var candidates = GetCandidateSchemas(schema)
.OrderBy(s => s.Name)
.GroupBy(s => s.Namespace)
.OrderBy(g => g.Key);
if (candidates.Count() < 1)
{
throw new UnsupportedSchemaException(schema, $"Code can only be generated for enums and records.");
}
var unit = SyntaxFactory.CompilationUnit();
foreach (var group in candidates)
{
var members = group
.Select(candidate => candidate switch
{
EnumSchema enumSchema => GenerateEnum(enumSchema) as MemberDeclarationSyntax,
RecordSchema recordSchema => GenerateClass(recordSchema) as MemberDeclarationSyntax,
_ => default,
/// <summary>
/// Generates a class declaration for a record schema.
/// </summary>
/// <param name="schema">
/// The schema to generate a class for.
/// </param>
/// <returns>
/// A class declaration with a property for each field of the record schema.
/// </returns>
/// <throws cref="UnsupportedSchemaException">
/// Thrown when a field schema is not recognized.
/// </throws>
public virtual ClassDeclarationSyntax GenerateClass(RecordSchema schema)
{
var declaration = SyntaxFactory.ClassDeclaration(schema.Name)
.AddModifiers(SyntaxFactory.Token(SyntaxKind.PublicKeyword))
.AddMembers(schema.Fields
.Select(field =>
{
var child = SyntaxFactory
.PropertyDeclaration(
GetPropertyType(field.Type),
field.Name
)
.AddModifiers(SyntaxFactory.Token(SyntaxKind.PublicKeyword))
.AddAccessorListAccessors(
SyntaxFactory.AccessorDeclaration(SyntaxKind.GetAccessorDeclaration)
.WithSemicolonToken(SyntaxFactory.Token(SyntaxKind.SemicolonToken)),
SyntaxFactory.AccessorDeclaration(SyntaxKind.SetAccessorDeclaration)
.WithSemicolonToken(SyntaxFactory.Token(SyntaxKind.SemicolonToken))
);
if (!string.IsNullOrEmpty(field.Documentation))
{
child = AddSummaryComment(child, field.Documentation);
}
return(child);
})
.Where(field => field != null)
.ToArray()
);
if (!string.IsNullOrEmpty(schema.Documentation))
{
declaration = AddSummaryComment(declaration, schema.Documentation);
}
return(declaration);
}
/// <summary>
/// Event handler that fires any time there is a change in a OPC monitored variable
/// </summary>
/// <param name="emitter"></param>
/// <param name="items"></param>
public void OnNotification(object emitter, MonItemNotificationArgs items)
{
RecordSchema schema = schemas.GetSchema(items.dataType);
if (schema != null)
{
foreach (var itm in items.values)
{
if (DataValue.IsBad(itm))
{
continue;
}
var m = buildKafkaMessage(itm, schema, items.dataType, items.name);
if (m == null)
{
continue;
}
// not waiting here
var status = sendMessage(_conf.opcSystemName, m);
log.Debug("Sending message {0}:{1} t:{2}", m.Key, m.Value, m.Timestamp.ToString());
}
}
}
internal Encoder.WriteItem Resolve(RecordSchema recordSchema)
{
WriteStep[] writeSteps = new WriteStep[recordSchema.Fields.Count];
int index = 0;
foreach (Field field in recordSchema)
{
var record = new WriteStep
{
WriteField = Resolver.ResolveWriter(field.Schema),
Field = field
};
writeSteps[index++] = record;
}
void RecordResolver(object v, IWriter e)
{
WriteRecordFields(v, writeSteps, e);
}
return(RecordResolver);
}
/// <summary>
/// Ctor.
/// </summary>
/// <param name="eventType">type to create</param>
/// <param name="eventAdapterService">event factory</param>
/// <param name="properties">written properties</param>
public EventBeanManufacturerAvro(
AvroSchemaEventType eventType,
EventAdapterService eventAdapterService,
IList<WriteablePropertyDescriptor> properties)
{
_eventAdapterService = eventAdapterService;
_eventType = (AvroEventType) eventType;
_schema = _eventType.SchemaAvro;
_indexPerWritable = new Field[properties.Count];
for (int i = 0; i < properties.Count; i++)
{
string propertyName = properties[i].PropertyName;
var field = _schema.GetField(propertyName);
if (field == null)
{
throw new IllegalStateException(
"Failed to find property '" + propertyName + "' among the array indexes");
}
_indexPerWritable[i] = field;
}
}
protected virtual IDictionary <string, object> ReadRecord(RecordSchema writerSchema, Schema readerSchema, IReader dec)
{
RecordSchema rs = (RecordSchema)readerSchema;
Record result = new Record(rs);
foreach (Field wf in writerSchema)
{
Field rf;
if (rs.TryGetFieldAlias(wf.Name, out rf))
{
object obj = null;
TryGetField(result, wf.Name, rf.Pos, out obj);
AddField(result, rf.aliases?[0] ?? wf.Name, rf.Pos, Read(wf.Schema, rf.Schema, dec));
}
else
{
Skip(wf.Schema, dec);
}
}
return(result.Contents);
}
public void TestPrimitiveWithMetadata(string rawSchema, Schema.Type type)
{
Schema definedSchema = Schema.Parse(rawSchema);
Assert.IsTrue(definedSchema is PrimitiveSchema);
Assert.AreEqual(type.ToString().ToLower(), definedSchema.Name);
Assert.AreEqual(type, definedSchema.Tag);
testEquality(rawSchema, definedSchema);
testToString(definedSchema);
Assert.True(definedSchema.ToString().Contains("metafield"));
var rawRecordSchema = "{\"type\":\"record\",\"name\":\"Foo\"," +
"\"fields\":[{\"name\":\"f1\",\"type\":" + rawSchema +
"}]}";
Schema baseRecordSchema = Schema.Parse(rawRecordSchema);
Assert.AreEqual(Schema.Type.Record, baseRecordSchema.Tag);
RecordSchema recordSchema = baseRecordSchema as RecordSchema;
Assert.IsNotNull(recordSchema);
Assert.AreEqual(1, recordSchema.Count);
Assert.IsTrue(recordSchema["f1"].Schema is PrimitiveSchema);
Assert.AreEqual(type.ToString().ToLower(), recordSchema["f1"].Schema.Name);
Assert.AreEqual(type, recordSchema["f1"].Schema.Tag);
testEquality(rawRecordSchema, baseRecordSchema);
testToString(recordSchema["f1"].Schema);
Assert.True(baseRecordSchema.ToString().Contains("metafield"));
Assert.True(recordSchema["f1"].Schema.ToString().Contains("metafield"));
Assert.True(definedSchema.Equals(recordSchema["f1"].Schema));
Assert.AreEqual(definedSchema.GetHashCode(), recordSchema["f1"].Schema.GetHashCode());
}
public void TestIt()
{
// Bean
RunAssertion(
"Bean", SupportEventInfra.FBEANWTYPE,
new Bean_Type_Root(),
new Bean_Type_1(),
new Bean_Type_2(),
new Bean_Type_2_1());
// Map
RunAssertion(
"Map", SupportEventInfra.FMAPWTYPE,
new Dictionary <string, object>(),
new Dictionary <string, object>(),
new Dictionary <string, object>(),
new Dictionary <string, object>());
// OA
RunAssertion(
"OA", SupportEventInfra.FOAWTYPE,
new object[0],
new object[0],
new object[0],
new object[0]);
// Avro
RecordSchema fake = SchemaBuilder.Record("fake");
RunAssertion(
"Avro", SupportEventInfra.FAVROWTYPE,
new GenericRecord(fake),
new GenericRecord(fake),
new GenericRecord(fake),
new GenericRecord(fake));
}
public void RecordWithParallelDependencies()
{
var node = new RecordSchema("Node");
node.Fields.Add(new RecordField("Value", new IntSchema()));
node.Fields.Add(new RecordField("Children", new ArraySchema(node)));
var schema = new RecordSchema("Reference");
schema.Fields.Add(new RecordField("Node", node));
schema.Fields.Add(new RecordField("RelatedNodes", new ArraySchema(node)));
var deserialize = deserializerBuilder.BuildDelegate <Reference>(schema);
var serialize = serializerBuilder.BuildDelegate <Reference>(schema);
using (stream)
{
serialize(
new Reference()
{
Node = new Node()
{
Children = Array.Empty <Node>(),
},
RelatedNodes = Array.Empty <Node>(),
},
new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
var root = deserialize(ref reader);
Assert.Empty(root.Node.Children);
Assert.Empty(root.RelatedNodes);
}
private static GenericRecord mkRecord(object[] kv, RecordSchema s)
{
GenericRecord input = new GenericRecord(s);
for (int i = 0; i < kv.Length; i += 2)
{
string fieldName = (string)kv[i];
object fieldValue = kv[i + 1];
Schema inner = s[fieldName].Schema;
if (inner is EnumSchema)
{
GenericEnum ge = new GenericEnum(inner as EnumSchema, (string)fieldValue);
fieldValue = ge;
}
else if (inner is FixedSchema)
{
GenericFixed gf = new GenericFixed(inner as FixedSchema);
gf.Value = (byte[])fieldValue;
fieldValue = gf;
}
input.Add(fieldName, fieldValue);
}
return(input);
}
private CsvToAvroGenericWriter(RecordSchema schema, string outputFilePath, Mode mode)
{
_avroSchema = schema;
BuildDataFileWriter(outputFilePath, mode);
}
/// <summary>
/// Gets an instance of CsvToAvroGenericWriter.
/// </summary>
/// <param name="schema">An Avro RecordSchema object containing the schema.</param>
/// <param name="outputFilePath">The path the Avro file should be written to.</param>
/// <param name="mode">
/// If the output Avro file already exists, specifies whether it should be overwritten or appended to.
/// The default is Overwrite.
/// </param>
/// <returns>A CsvToAvroGenericWriter object.</returns>
public static CsvToAvroGenericWriter CreateFromSchema(RecordSchema schema, string outputFilePath,
Mode mode = Mode.Overwrite)
{
return(new CsvToAvroGenericWriter(schema, outputFilePath, mode));
}
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
DeliveryResult <Null, GenericRecord> dr;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, GenericRecord>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Value = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, User>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
var user = new User
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <Null, User> {
Value = user
}).Wait();
}
Assert.Null(dr.Message.Key);
Assert.NotNull(dr.Message.Value);
dr.Message.Value.TryGetValue("name", out object name);
dr.Message.Value.TryGetValue("favorite_number", out object number);
dr.Message.Value.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, GenericRecord>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.Build())
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out object msgName);
record.Message.Value.TryGetValue("favorite_number", out object msgNumber);
record.Message.Value.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out msgName);
record.Message.Value.TryGetValue("favorite_number", out msgNumber);
record.Message.Value.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, User>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Assert.Equal("my name 2", record.Message.Value.name);
Assert.Equal(44, record.Message.Value.favorite_number);
Assert.Null(record.Message.Value.favorite_color);
}
}
public override void WriteRequest(RecordSchema schema, object request, Encoder encoder)
{
new GenericWriter<object>(schema).Write(request, encoder);
}
public IEnumerator GetEnumerator()
{
var record01 = new RecordSchema()
{
Name = "TestRecord01"
};
var record02 = new RecordSchema()
{
Name = "TestRecord02"
};
var errord01 = new ErrorSchema()
{
Name = "TestError01"
};
var errord02 = new ErrorSchema()
{
Name = "TestError02"
};
var protocol01 = new AvroProtocol
{
Name = "ProtocolName"
};
yield return(new object[] {
protocol01,
@"{""protocol"":""ProtocolName""}",
@"{ ""protocol"": ""ProtocolName"" }",
@"{ ""namespace"": """", ""protocol"": ""ProtocolName"", ""doc"": """", ""types"": [], ""messages"": {} }"
});
var protocol02 = new AvroProtocol
{
Name = "ProtocolName",
Namespace = "Test.Namespace"
};
yield return(new object[] {
protocol02,
@"{""protocol"":""Test.Namespace.ProtocolName""}",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"" }",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": """", ""types"": [], ""messages"": {} }"
});
var protocol03 = new AvroProtocol
{
Name = "ProtocolName",
Namespace = "Test.Namespace",
Doc = "Test Documentation"
};
yield return(new object[] {
protocol03,
@"{""protocol"":""Test.Namespace.ProtocolName""}",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"" }",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": [], ""messages"": {} }"
});
var protocol04 = new AvroProtocol
{
Name = "ProtocolName",
Namespace = "Test.Namespace",
Doc = "Test Documentation"
};
protocol04.AddType(record01);
protocol04.AddType(record02);
protocol04.AddType(errord01);
protocol04.AddType(errord02);
yield return(new object[] {
protocol04,
@"{""protocol"":""Test.Namespace.ProtocolName"",""types"":" +
@"[{""name"":""Test.Namespace.TestRecord01"",""type"":""record"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestRecord02"",""type"":""record"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestError01"",""type"":""error"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestError02"",""type"":""error"",""fields"":[]}" +
@"]}",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " +
@"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""fields"": [] }, " +
@"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError01"", ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError02"", ""fields"": [] }" +
@"] }",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " +
@"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] " +
@"}]," +
@" ""messages"": {} }"
});
var protocol05 = new AvroProtocol
{
Name = "ProtocolName",
Namespace = "Test.Namespace",
Doc = "Test Documentation"
};
protocol05.AddType(record01);
protocol05.AddType(record02);
protocol05.AddType(errord01);
protocol05.AddType(errord02);
var message01 = new MessageSchema("M01")
{
Doc = "Test Doc 01"
};
message01.AddParameter(new ParameterSchema("p01", record01));
message01.AddParameter(new ParameterSchema("p02", record02));
message01.AddError(errord01);
message01.AddError(errord02);
message01.Response = new DoubleSchema();
var message02 = new MessageSchema("M02");
message02.AddParameter(new ParameterSchema("p01", record01));
message02.AddError(errord01);
message02.Oneway = true;
protocol05.AddMessage(message01);
protocol05.AddMessage(message02);
yield return(new object[] {
protocol05,
@"{""protocol"":""Test.Namespace.ProtocolName"",""types"":" +
@"[{""name"":""Test.Namespace.TestRecord01"",""type"":""record"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestRecord02"",""type"":""record"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestError01"",""type"":""error"",""fields"":[]}," +
@"{""name"":""Test.Namespace.TestError02"",""type"":""error"",""fields"":[]}" +
@"]," +
@"""messages"":{" +
@"""M01"":{" +
@"""request"":[{""name"":""p01"",""type"":""Test.Namespace.TestRecord01""},{""name"":""p02"",""type"":""Test.Namespace.TestRecord02""}]," +
@"""response"":""double""," +
@"""errors"":[""Test.Namespace.TestError01"",""Test.Namespace.TestError02""]" +
@"}," +
@"""M02"":{" +
@"""request"":[{""name"":""p01"",""type"":""Test.Namespace.TestRecord01""}]," +
@"""errors"":[""Test.Namespace.TestError01""]," +
@"""one-way"":true" +
@"}}}",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " +
@"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""fields"": [] }, " +
@"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError01"", ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError02"", ""fields"": [] }" +
@"], " +
@"""messages"": {" +
@"""M01"": {" +
@"""doc"": ""Test Doc 01"", " +
@"""request"": [{ ""name"": ""p01"", ""type"": ""TestRecord01"" }, { ""name"": ""p02"", ""type"": ""TestRecord02"" }], " +
@"""response"": ""double"", " +
@"""errors"": [""TestError01"", ""TestError02""]" +
@"}, " +
@"""M02"": {" +
@"""request"": [{ ""name"": ""p01"", ""type"": ""TestRecord01"" }], " +
@"""errors"": [""TestError01""], " +
@"""one-way"": true" +
@"}} }",
@"{ ""namespace"": ""Test.Namespace"", ""protocol"": ""ProtocolName"", ""doc"": ""Test Documentation"", ""types"": " +
@"[{ ""type"": ""record"", ""name"": ""TestRecord01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""record"", ""name"": ""TestRecord02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError01"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }, " +
@"{ ""type"": ""error"", ""name"": ""TestError02"", ""namespace"": ""Test.Namespace"", ""doc"": """", ""aliases"": [], ""fields"": [] }" +
@"], " +
@"""messages"": {" +
@"""M01"": {" +
@"""doc"": ""Test Doc 01"", " +
@"""request"": [{ ""name"": ""p01"", ""type"": ""Test.Namespace.TestRecord01"" }, { ""name"": ""p02"", ""type"": ""Test.Namespace.TestRecord02"" }], " +
@"""response"": { ""type"": ""double"" }, " +
@"""errors"": [""Test.Namespace.TestError01"", ""Test.Namespace.TestError02""], " +
@"""one-way"": false" +
@"}, " +
@"""M02"": {" +
@"""doc"": """", " +
@"""request"": [{ ""name"": ""p01"", ""type"": ""Test.Namespace.TestRecord01"" }], " +
@"""response"": null, " +
@"""errors"": [""Test.Namespace.TestError01""], " +
@"""one-way"": true" +
@"}} }"
});
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
string groupName = "avro-generic-example-group";
// var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = groupName
})
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry))
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// commit final offsets and leave the group.
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var producer =
new ProducerBuilder <string, GenericRecord>(new ProducerConfig {
BootstrapServers = bootstrapServers
})
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
await producer
.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
public bool SchemaMatches(RecordSchema schema)
{
return(_schema.Equals(schema));
}
/// <summary>
/// Reads the passed JToken default value field and writes it in the specified encoder
/// </summary>
/// <param name="enc">encoder to use for writing</param>
/// <param name="schema">schema object for the current field</param>
/// <param name="jtok">default value as JToken</param>
public static void EncodeDefaultValue(Encoder enc, Schema schema, JToken jtok)
{
if (null == jtok)
{
return;
}
switch (schema.Tag)
{
case Schema.Type.Boolean:
if (jtok.Type != JTokenType.Boolean)
{
throw new AvroException("Default boolean value " + jtok.ToString() + " is invalid, expected is json boolean.");
}
enc.WriteBoolean((bool)jtok);
break;
case Schema.Type.Int:
if (jtok.Type != JTokenType.Integer)
{
throw new AvroException("Default int value " + jtok.ToString() + " is invalid, expected is json integer.");
}
enc.WriteInt(Convert.ToInt32((int)jtok));
break;
case Schema.Type.Long:
if (jtok.Type != JTokenType.Integer)
{
throw new AvroException("Default long value " + jtok.ToString() + " is invalid, expected is json integer.");
}
enc.WriteLong(Convert.ToInt64((long)jtok));
break;
case Schema.Type.Float:
if (jtok.Type != JTokenType.Float)
{
throw new AvroException("Default float value " + jtok.ToString() + " is invalid, expected is json number.");
}
enc.WriteFloat((float)jtok);
break;
case Schema.Type.Double:
if (jtok.Type == JTokenType.Integer)
{
enc.WriteDouble(Convert.ToDouble((int)jtok));
}
else if (jtok.Type == JTokenType.Float)
{
enc.WriteDouble(Convert.ToDouble((float)jtok));
}
else
{
throw new AvroException("Default double value " + jtok.ToString() + " is invalid, expected is json number.");
}
break;
case Schema.Type.Bytes:
if (jtok.Type != JTokenType.String)
{
throw new AvroException("Default bytes value " + jtok.ToString() + " is invalid, expected is json string.");
}
var en = System.Text.Encoding.GetEncoding("iso-8859-1");
enc.WriteBytes(en.GetBytes((string)jtok));
break;
case Schema.Type.Fixed:
if (jtok.Type != JTokenType.String)
{
throw new AvroException("Default fixed value " + jtok.ToString() + " is invalid, expected is json string.");
}
en = System.Text.Encoding.GetEncoding("iso-8859-1");
int len = (schema as FixedSchema).Size;
byte[] bb = en.GetBytes((string)jtok);
if (bb.Length != len)
{
throw new AvroException("Default fixed value " + jtok.ToString() + " is not of expected length " + len);
}
enc.WriteFixed(bb);
break;
case Schema.Type.String:
if (jtok.Type != JTokenType.String)
{
throw new AvroException("Default string value " + jtok.ToString() + " is invalid, expected is json string.");
}
enc.WriteString((string)jtok);
break;
case Schema.Type.Enumeration:
if (jtok.Type != JTokenType.String)
{
throw new AvroException("Default enum value " + jtok.ToString() + " is invalid, expected is json string.");
}
enc.WriteEnum((schema as EnumSchema).Ordinal((string)jtok));
break;
case Schema.Type.Null:
if (jtok.Type != JTokenType.Null)
{
throw new AvroException("Default null value " + jtok.ToString() + " is invalid, expected is json null.");
}
enc.WriteNull();
break;
case Schema.Type.Array:
if (jtok.Type != JTokenType.Array)
{
throw new AvroException("Default array value " + jtok.ToString() + " is invalid, expected is json array.");
}
JArray jarr = jtok as JArray;
enc.WriteArrayStart();
enc.SetItemCount(jarr.Count);
foreach (JToken jitem in jarr)
{
enc.StartItem();
EncodeDefaultValue(enc, (schema as ArraySchema).ItemSchema, jitem);
}
enc.WriteArrayEnd();
break;
case Schema.Type.Record:
case Schema.Type.Error:
if (jtok.Type != JTokenType.Object)
{
throw new AvroException("Default record value " + jtok.ToString() + " is invalid, expected is json object.");
}
RecordSchema rcs = schema as RecordSchema;
JObject jo = jtok as JObject;
foreach (Field field in rcs)
{
JToken val = jo[field.Name];
if (null == val)
{
val = field.DefaultValue;
}
if (null == val)
{
throw new AvroException("No default value for field " + field.Name);
}
EncodeDefaultValue(enc, field.Schema, val);
}
break;
case Schema.Type.Map:
if (jtok.Type != JTokenType.Object)
{
throw new AvroException("Default map value " + jtok.ToString() + " is invalid, expected is json object.");
}
jo = jtok as JObject;
enc.WriteMapStart();
enc.SetItemCount(jo.Count);
foreach (KeyValuePair <string, JToken> jp in jo)
{
enc.StartItem();
enc.WriteString(jp.Key);
EncodeDefaultValue(enc, (schema as MapSchema).ValueSchema, jp.Value);
}
enc.WriteMapEnd();
break;
case Schema.Type.Union:
enc.WriteUnionIndex(0);
EncodeDefaultValue(enc, (schema as UnionSchema).Schemas[0], jtok);
break;
default:
throw new AvroException("Unsupported schema type " + schema.Tag);
}
}
public void RecordWithMissingFields()
{
var boolean = new BooleanSchema();
var array = new ArraySchema(boolean);
var map = new MapSchema(new IntSchema());
var @enum = new EnumSchema("Ordinal", new[] { "None", "First", "Second", "Third", "Fourth" });
var union = new UnionSchema(new Schema[]
{
new NullSchema(),
array,
});
var schema = new RecordSchema("AllFields")
{
Fields = new[]
{
new RecordField("First", union),
new RecordField("Second", union),
new RecordField("Third", array),
new RecordField("Fourth", array),
new RecordField("Fifth", map),
new RecordField("Sixth", map),
new RecordField("Seventh", @enum),
new RecordField("Eighth", @enum),
},
};
var deserialize = deserializerBuilder.BuildDelegate <WithoutEvenFields>(schema);
var serialize = serializerBuilder.BuildDelegate <WithEvenFields>(schema);
var value = new WithEvenFields()
{
First = new List <bool>()
{
false
},
Second = new List <bool>()
{
false, false
},
Third = new List <bool>()
{
false, false, false
},
Fourth = new List <bool>()
{
false
},
Fifth = new Dictionary <string, int>()
{
{ "first", 1 }
},
Sixth = new Dictionary <string, int>()
{
{ "first", 1 }, { "second", 2 }
},
Seventh = ImplicitEnum.First,
Eighth = ImplicitEnum.None,
};
using (stream)
{
serialize(value, new Utf8JsonWriter(stream));
}
var reader = new Utf8JsonReader(stream.ToArray());
Assert.Equal(value.Seventh, deserialize(ref reader).Seventh);
}
protected override RecordAccess GetRecordAccess(RecordSchema schema)
{
return(new GenericRecordAccess(schema));
}
protected abstract void EnsureRecordObject(RecordSchema recordSchema, object value);
public GenericRecord(RecordSchema schema)
{
this.Schema = schema;
}
protected override object ReadRecord(object reuse, RecordSchema writerSchema, Schema readerSchema, Decoder decoder)
{
reuse = reuse ?? CreateInstance(readerSchema, Schema.Type.Record);
return(base.ReadRecord(reuse, writerSchema, readerSchema, decoder));
}
public SpecificRecordAccess(RecordSchema readerSchema)
{
objCreator = GetConstructor(readerSchema.Fullname, Schema.Type.Record);
}
private ReadItem ResolveRecord(RecordSchema writerSchema, RecordSchema readerSchema)
{
var schemaPair = new SchemaPair(writerSchema, readerSchema);
ReadItem recordReader;
if (_recordReaders.TryGetValue(schemaPair, out recordReader))
{
return(recordReader);
}
FieldReader[] fieldReaderArray = null;
var recordAccess = GetRecordAccess(readerSchema);
recordReader = (r, d) => ReadRecord(r, d, recordAccess, fieldReaderArray);
_recordReaders.Add(schemaPair, recordReader);
var readSteps = new List <FieldReader>();
foreach (Field wf in writerSchema)
{
Field rf;
if (readerSchema.TryGetFieldAlias(wf.Name, out rf))
{
var readItem = ResolveReader(wf.Schema, rf.Schema);
if (IsReusable(rf.Schema.Tag))
{
readSteps.Add((rec, d) => recordAccess.AddField(rec, rf.Name, rf.Pos,
readItem(recordAccess.GetField(rec, rf.Name, rf.Pos), d)));
}
else
{
readSteps.Add((rec, d) => recordAccess.AddField(rec, rf.Name, rf.Pos,
readItem(null, d)));
}
}
else
{
var skip = GetSkip(wf.Schema);
readSteps.Add((rec, d) => skip(d));
}
}
// fill in defaults for any reader fields not in the writer schema
foreach (Field rf in readerSchema)
{
if (writerSchema.Contains(rf.Name))
{
continue;
}
var defaultStream = new MemoryStream();
var defaultEncoder = new BinaryEncoder(defaultStream);
defaultStream.Position = 0; // reset for writing
Resolver.EncodeDefaultValue(defaultEncoder, rf.Schema, rf.DefaultValue);
defaultStream.Flush();
var defaultBytes = defaultStream.ToArray();
var readItem = ResolveReader(rf.Schema, rf.Schema);
var rfInstance = rf;
if (IsReusable(rf.Schema.Tag))
{
readSteps.Add((rec, d) => recordAccess.AddField(rec, rfInstance.Name, rfInstance.Pos,
readItem(recordAccess.GetField(rec, rfInstance.Name, rfInstance.Pos),
new BinaryDecoder(new MemoryStream(defaultBytes)))));
}
else
{
readSteps.Add((rec, d) => recordAccess.AddField(rec, rfInstance.Name, rfInstance.Pos,
readItem(null, new BinaryDecoder(new MemoryStream(defaultBytes)))));
}
}
fieldReaderArray = readSteps.ToArray();
return(recordReader);
}
public GenericRecordAccess(RecordSchema schema)
{
_schema = schema;
}
protected abstract RecordAccess GetRecordAccess(RecordSchema readerSchema);