private async Task SendMessageAvro(string message)
{
var config = new ProducerConfig
{
BootstrapServers = "kafka:29092",
ClientId = Dns.GetHostName()
};
String key = "key1";
String userSchema = "{\"type\":\"record\"," +
"\"name\":\"myrecord\"," +
"\"fields\":[{\"name\":\"f1\",\"type\":\"string\"}]}";
var schema = (RecordSchema)RecordSchema.Parse(userSchema);
GenericRecord avroRecord = new GenericRecord(schema);
avroRecord.Add("f1", "value");
var schemaRegistryUrl = "http://schema-registry:8085";
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = schemaRegistryUrl
}))
{
using (var producer = new ProducerBuilder <string, GenericRecord>(config)
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
await producer.ProduceAsync("api-methods-requested-avro",
new Message <string, GenericRecord> {
Key = Guid.NewGuid().ToString("N"), Value = avroRecord
});
}
}
}
/**
* This is an example function that creates an Avro schema object based on pre-defined Avro JSON-like schema. It then
* adds some example data into the record fields, creates a buffer associated with the record, and sends the buffer
* out through UDP connection.
**/
public static void sender()
{
/**
* Creating a schema object by loading schema object file (.avsc)
* Example.AVSC looks like this:
*
* {"namespace": "example.avro",
* "type": "record",
* "name": "User",
* "fields": [
* {"name": "name", "type": "string"},
* {"name": "favorite_number", "type": ["int", "null"]},
* {"name": "favorite_color", "type": ["string", "null"]}
* ]
* }
**/
var schema = RecordSchema.Parse(File.ReadAllText(@"C:\Users\user\src\example.avsc")) as RecordSchema;
//Passing in schema object to get a record object
var exampleRecorder = new GenericRecord(schema);
//Filling out records with the corresponding schema
exampleRecorder.Add("name", "myExample");
exampleRecorder.Add("favorite_number", 999);
exampleRecorder.Add("favorite_color", "read");
//Creating an Avro buffer stream
ByteBufferOutputStream buffer = new ByteBufferOutputStream();
//Wraping the buffer stream with the encoder that does low level serialization
Avro.IO.Encoder encoder = new BinaryEncoder(buffer);
//Creating a writer with the corresponding schema object
var writer = new DefaultWriter(schema);
//Write (serialize) record object into buffer outputStream with encoder
writer.Write <GenericRecord>(exampleRecorder, encoder);
//And flush
buffer.Flush();
//Creating a UDP client
UdpClient udpClient = new UdpClient(0);
//Connect to endpoint with host and port number arguments
udpClient.Connect("my_udp_end_point.com", 9999);
//Get buffer list from buffer stream
List <MemoryStream> bufferList = buffer.GetBufferList();
//For each memory stream, creating a byte array, and deliver the byte array to endpoint
//You actually do not need a foreach loop, because you will only have one memory stream
foreach (MemoryStream ms in bufferList)
{
byte[] bufferArray;
bufferArray = ms.ToArray();
udpClient.Send(bufferArray, bufferArray.Length);
}
udpClient.Close();
}
static AvroEventFormatter()
{
// we're going to confidently assume that the embedded schema works. If not, type initialization
// will fail and that's okay since the type is useless without the proper schema
using (var sr = new StreamReader(typeof(AvroEventFormatter).Assembly.GetManifestResourceStream("CloudNative.CloudEvents.Avro.AvroSchema.json")))
{
avroSchema = (RecordSchema)RecordSchema.Parse(sr.ReadToEnd());
}
avroReader = new DefaultReader(avroSchema, avroSchema);
avroWriter = new DefaultWriter(avroSchema);
}
public static void ProduceGenericMultipleTopics(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
var topic2 = Guid.NewGuid().ToString();
DeliveryResult <Null, GenericRecord> dr;
DeliveryResult <Null, GenericRecord> dr2;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, GenericRecord>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
dr2 = p.ProduceAsync(topic2, new Message <Null, GenericRecord> {
Key = null, Value = record
}).Result;
}
Assert.Null(dr.Key);
Assert.NotNull(dr.Value);
Assert.Null(dr2.Key);
Assert.NotNull(dr2.Value);
}
public async Task <string> PublishGenericAvroMessage(PublishSingleRequest request)
{
var schema = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Kafka.Workshop.Avro"",
""type"": ""record"",
""name"": ""JustAMessage"",
""fields"": [
{""name"": ""title"", ""type"": ""string""},
{""name"": ""message"", ""type"": ""string""}
]
}"
);
var cts = new CancellationTokenSource();
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig
{
Url = _schemaRegistryUrl
}))
using (var producer =
new ProducerBuilder <string, GenericRecord>(new ProducerConfig
{
BootstrapServers = _defaultBrokerString,
SocketTimeoutMs = 5000,
MessageTimeoutMs = 3000,
RequestTimeoutMs = 3000,
})
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
var record = new GenericRecord(schema);
record.Add("title", request.Message.Title);
record.Add("message", request.Message.Message);
var res = await producer
.ProduceAsync(request.Topic, new Message <string, GenericRecord> {
Key = "producer", Value = record
})
.ContinueWith(task => task.IsFaulted
? $"Error producing message {task.Exception.Message}"
: $"Message sent to: {task.Result.TopicPartitionOffset}", cts.Token);
return(res);
}
}
public AvroSerializer(string schema, bool isPath)
{
if (isPath)
{
var fullPath = Path.GetFullPath(schema);
if (File.Exists(fullPath))
{
Schema = Avro.RecordSchema.Parse(File.ReadAllText(fullPath));
}
}
else
{
if (!string.IsNullOrEmpty(schema))
{
Schema = RecordSchema.Parse(schema);
}
}
}
public static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers)
{
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryServers
};
var topic = Guid.NewGuid().ToString();
DeliveryResult <Null, GenericRecord> dr;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, GenericRecord>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
var record = new GenericRecord(s);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <Null, GenericRecord> {
Value = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <Null, User>(config)
.SetKeySerializer(Serializers.Null)
.SetValueSerializer(new AvroSerializer <User>(schemaRegistry))
.Build())
{
var user = new User
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <Null, User> {
Value = user
}).Wait();
}
Assert.Null(dr.Message.Key);
Assert.NotNull(dr.Message.Value);
dr.Message.Value.TryGetValue("name", out object name);
dr.Message.Value.TryGetValue("favorite_number", out object number);
dr.Message.Value.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, GenericRecord>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.Build())
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out object msgName);
record.Message.Value.TryGetValue("favorite_number", out object msgNumber);
record.Message.Value.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Value.TryGetValue("name", out msgName);
record.Message.Value.TryGetValue("favorite_number", out msgNumber);
record.Message.Value.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <Null, User>(cconfig)
.SetKeyDeserializer(Deserializers.Null)
.SetValueDeserializer(new AvroDeserializer <User>(schemaRegistry).AsSyncOverAsync())
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Assert.Equal("my name 2", record.Message.Value.name);
Assert.Equal(44, record.Message.Value.favorite_number);
Assert.Null(record.Message.Value.favorite_color);
}
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
string groupName = "avro-generic-example-group";
// var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = groupName
})
.SetKeyDeserializer(new AvroDeserializer <string>(schemaRegistry))
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry))
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// commit final offsets and leave the group.
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
SchemaRegistryUrl = schemaRegistryUrl
}))
using (var producer =
new ProducerBuilder <string, GenericRecord>(new ProducerConfig {
BootstrapServers = bootstrapServers
})
.SetKeySerializer(new AvroSerializer <string>(schemaRegistry))
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
int i = 0;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
await producer
.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
})
.ContinueWith(task => task.IsFaulted
? $"error producing message: {task.Exception.Message}"
: $"produced to: {task.Result.TopicPartitionOffset}");
}
}
cts.Cancel();
}
static async Task Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("Usage: .. bootstrapServers schemaRegistryUrl topicName");
return;
}
string bootstrapServers = args[0];
string schemaRegistryUrl = args[1];
string topicName = args[2];
string groupName = "avro-generic-example-group";
// var s = (RecordSchema)RecordSchema.Parse(File.ReadAllText("my-schema.json"));
var s = (RecordSchema)RecordSchema.Parse(
@"{
""type"": ""record"",
""name"": ""User"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": ""long""},
{""name"": ""favorite_color"", ""type"": ""string""}
]
}"
);
CancellationTokenSource cts = new CancellationTokenSource();
var consumeTask = Task.Run(() =>
{
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = schemaRegistryUrl
}))
using (var consumer =
new ConsumerBuilder <string, GenericRecord>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = groupName
})
.SetValueDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.SetErrorHandler((_, e) => Console.WriteLine($"Error: {e.Reason}"))
.Build())
{
consumer.Subscribe(topicName);
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(cts.Token);
Console.WriteLine($"Key: {consumeResult.Message.Key}\nValue: {consumeResult.Message.Value}");
}
catch (ConsumeException e)
{
Console.WriteLine($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
// commit final offsets and leave the group.
consumer.Close();
}
}
});
using (var schemaRegistry = new CachedSchemaRegistryClient(new SchemaRegistryConfig {
Url = schemaRegistryUrl
}))
using (var producer =
new ProducerBuilder <string, GenericRecord>(new ProducerConfig {
BootstrapServers = bootstrapServers
})
.SetValueSerializer(new AvroSerializer <GenericRecord>(schemaRegistry))
.Build())
{
Console.WriteLine($"{producer.Name} producing on {topicName}. Enter user names, q to exit.");
long i = 1;
string text;
while ((text = Console.ReadLine()) != "q")
{
var record = new GenericRecord(s);
record.Add("name", text);
record.Add("favorite_number", i++);
record.Add("favorite_color", "blue");
try
{
var dr = await producer.ProduceAsync(topicName, new Message <string, GenericRecord> {
Key = text, Value = record
});
Console.WriteLine($"produced to: {dr.TopicPartitionOffset}");
}
catch (ProduceException <string, GenericRecord> ex)
{
// In some cases (notably Schema Registry connectivity issues), the InnerException
// of the ProduceException contains additional informatiom pertaining to the root
// cause of the problem. This information is automatically included in the output
// of the ToString() method of the ProduceException, called implicitly in the below.
Console.WriteLine($"error producing message: {ex}");
}
}
}
cts.Cancel();
}
private static void ProduceConsumeGeneric(string bootstrapServers, string schemaRegistryServers, SubjectNameStrategy nameStrategy)
{
var rs = (RecordSchema)RecordSchema.Parse(
@"{
""namespace"": ""Confluent.Kafka.Examples.AvroSpecific"",
""type"": ""record"",
""name"": ""ProduceConsumeUser2"",
""fields"": [
{""name"": ""name"", ""type"": ""string""},
{""name"": ""favorite_number"", ""type"": [""int"", ""null""]},
{""name"": ""favorite_color"", ""type"": [""string"", ""null""]}
]
}"
);
var config = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var schemaRegistryConfig = new SchemaRegistryConfig
{
Url = schemaRegistryServers,
};
var avroSerializerConfig = new AvroSerializerConfig
{
SubjectNameStrategy = nameStrategy
};
var topic = Guid.NewGuid().ToString();
DeliveryResult <GenericRecord, Null> dr;
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <GenericRecord, Null>(config)
// Test KeySubjectNameStrategy here,
// and ValueSubjectNameStrategy in ProduceConsume.
.SetKeySerializer(new AvroSerializer <GenericRecord>(schemaRegistry, avroSerializerConfig))
.SetValueSerializer(Serializers.Null)
.Build())
{
var record = new GenericRecord(rs);
record.Add("name", "my name 2");
record.Add("favorite_number", 44);
record.Add("favorite_color", null);
dr = p.ProduceAsync(topic, new Message <GenericRecord, Null> {
Key = record
}).Result;
}
// produce a specific record (to later consume back as a generic record).
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var p =
new ProducerBuilder <ProduceConsumeUser2, Null>(config)
.SetKeySerializer(new AvroSerializer <ProduceConsumeUser2>(schemaRegistry, avroSerializerConfig))
.SetValueSerializer(Serializers.Null)
.Build())
{
var user = new ProduceConsumeUser2
{
name = "my name 3",
favorite_number = 47,
favorite_color = "orange"
};
p.ProduceAsync(topic, new Message <ProduceConsumeUser2, Null> {
Key = user
}).Wait();
}
Assert.Null(dr.Message.Value);
Assert.NotNull(dr.Message.Key);
dr.Message.Key.TryGetValue("name", out object name);
dr.Message.Key.TryGetValue("favorite_number", out object number);
dr.Message.Key.TryGetValue("favorite_color", out object color);
Assert.IsType <string>(name);
Assert.IsType <int>(number);
Assert.Equal("my name 2", name);
Assert.Equal(44, number);
Assert.Null(color);
var cconfig = new ConsumerConfig {
GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers
};
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <GenericRecord, Null>(cconfig)
.SetKeyDeserializer(new AvroDeserializer <GenericRecord>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Null)
.Build())
{
// consume generic record produced as a generic record.
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Key.TryGetValue("name", out object msgName);
record.Message.Key.TryGetValue("favorite_number", out object msgNumber);
record.Message.Key.TryGetValue("favorite_color", out object msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.Equal("my name 2", msgName);
Assert.Equal(44, msgNumber);
Assert.Null(msgColor);
// consume generic record produced as a specific record.
record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
record.Message.Key.TryGetValue("name", out msgName);
record.Message.Key.TryGetValue("favorite_number", out msgNumber);
record.Message.Key.TryGetValue("favorite_color", out msgColor);
Assert.IsType <string>(msgName);
Assert.IsType <int>(msgNumber);
Assert.IsType <string>(msgColor);
Assert.Equal("my name 3", msgName);
Assert.Equal(47, msgNumber);
Assert.Equal("orange", msgColor);
}
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
using (var consumer =
new ConsumerBuilder <ProduceConsumeUser2, Null>(cconfig)
.SetKeyDeserializer(new AvroDeserializer <ProduceConsumeUser2>(schemaRegistry).AsSyncOverAsync())
.SetValueDeserializer(Deserializers.Null)
.Build())
{
consumer.Assign(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic, 0, dr.Offset)
});
var record = consumer.Consume(new CancellationTokenSource(TimeSpan.FromSeconds(10)).Token);
Assert.Equal("my name 2", record.Message.Key.name);
Assert.Equal(44, record.Message.Key.favorite_number);
Assert.Null(record.Message.Key.favorite_color);
}
// Check that what's in schema registry is what's expected.
using (var schemaRegistry = new CachedSchemaRegistryClient(schemaRegistryConfig))
{
var subjects = schemaRegistry.GetAllSubjectsAsync().Result;
if (nameStrategy == SubjectNameStrategy.TopicRecord)
{
Assert.Single(subjects.Where(s => s.Contains(topic)));
Assert.Single(subjects.Where(s => s == $"{topic}-{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
}
if (nameStrategy == SubjectNameStrategy.Topic)
{
Assert.Single(subjects.Where(s => s.Contains(topic)));
Assert.Single(subjects.Where(s => s == $"{topic}-key"));
}
if (nameStrategy == SubjectNameStrategy.Record)
{
Assert.Single(subjects.Where(s => s.Contains(topic))); // the string key.
Assert.Single(subjects.Where(s => s == $"{((Avro.RecordSchema)ProduceConsumeUser2._SCHEMA).Fullname}"));
}
}
}
} // end createSchemaFromString()
/// <summary>
/// Create an avro schema from either the columns or the schema string.
/// </summary>
private void createSchema()
{
// First, check if the schema has already been created
if (m_data.schema != null)
{
// nothing to do
return;
}
// Check if the schema string exists, if so, create the schema from that
if (m_data.schemaString != null)
{
try
{
m_data.schema = RecordSchema.Parse(m_data.schemaString);
return;
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
} // done creating the schema from the schema string
// Since the shortcuts didn't apply, create a JSON object from the columns
// and then create the schema and the schema string off it
// --------------------------------------------------------------------------
// Create the json string for the type
string schema_string = "";
// Create the json string opening with empty fields (with a generic 'type_name' (because the
// server always replaces the name with this string anyway) )
string schema_opening = "{'type':'record','name':'type_name','fields':[";
// Create the json string closing
string schema_closing = "]}";
schema_string += schema_opening;
// Create the json substrings for the columns
foreach (var column in m_data.columns)
{
// Add the name
string field_name = ("'name':'" + column.getName() + "'");
// Add the type
string field_type = "";
if (column.isNullable())
{ // the column is nullable, so we need a union
field_type = ("['" + column.getTypeString() + "','null']");
}
else // regular type, no union needed
{
field_type = ("'" + column.getTypeString() + "'");
}
field_type = ("'type':" + field_type);
// Put the field together
string field = ("{" + field_name + "," + field_type + "},");
schema_string += field;
} // end looping over the fields
// Trim the trailing comma from the fields
char[] comma = { ',' };
schema_string = schema_string.TrimEnd(comma);
// Add the ending of the json string
schema_string += schema_closing;
// Create the RecordSchema from the JSON string
try
{
m_data.schema = RecordSchema.Parse(schema_string);
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// Save the schema string
m_data.schemaString = m_data.schema.ToString();
return;
} // end createSchema()
} // end initialize()
/// <summary>
/// Creates a schema object from a string.
/// </summary>
/// <param name="typeSchema">The schema in a string format.</param>
/// <param name="properties">Properties for the columns.</param>
private void createSchemaFromString(string typeSchema,
IDictionary <string, IList <string> > properties = null)
{
// Create the avro schema from the string and save it
try
{
m_data.schema = RecordSchema.Parse(typeSchema);
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
var root = JObject.Parse(typeSchema);
var rootType = root["type"];
if ((null == rootType) || !rootType.ToString().Contains("record"))
{
throw new ArgumentException("Schema must be of type record.");
}
var fields = root["fields"];
if ((null == fields) || !fields.HasValues)
{
throw new ArgumentException("Schema has no fields.");
}
foreach (var field in fields)
{
//if (!field->first.empty() || field->second.empty())
//{
// throw std::invalid_argument("Schema has invalid field.");
//}
// Do NOT use ToString 'cause it includes the double quotes (turns it into a JSON representation)
var fieldName = (string)field["name"];
if (string.IsNullOrEmpty(fieldName))
{
throw new ArgumentException("Schema has unnamed field.");
}
if (m_data.columnMap.ContainsKey(fieldName))
{
throw new ArgumentException($"Duplicate field name {fieldName}.");
}
var fieldType = field["type"];
if (null == fieldType)
{
throw new ArgumentException($"Field {fieldName} has no type.");
}
// Flag for nullability
bool is_column_nullable = false;
if (fieldType.HasValues) // If it has children
{
var fieldTypeArray = fieldType;
foreach (var fieldTypeElement in fieldTypeArray.Children())
{
bool valid = false;
//if (fieldTypeElement->first.empty())
{
var fieldTypeElementString = fieldTypeElement.ToString();
if (!string.IsNullOrEmpty(fieldTypeElementString))
{
if (fieldTypeElementString == "null" || fieldTypeElementString == "\"null\"")
{
is_column_nullable = true;
valid = true;
}
else //if (fieldType->empty())
{
fieldType = fieldTypeElement; // fieldTypeElementString;
valid = true;
}
}
}
if (!valid)
{
throw new ArgumentException("Field {fieldName} has invalid type.");
}
}
//if (fieldType->empty())
//{
// throw new ArgumentException("Field " + *fieldName + " has invalid type.");
//}
}
Column.ColumnType columnType;
if (fieldType.ToString().Equals("bytes") || fieldType.ToString().Equals("\"bytes\""))
{
columnType = Column.ColumnType.BYTES;
}
else if (fieldType.ToString().Equals("double") || fieldType.ToString().Equals("\"double\""))
{
columnType = Column.ColumnType.DOUBLE;
}
else if (fieldType.ToString().Equals("float") || fieldType.ToString().Equals("\"float\""))
{
columnType = Column.ColumnType.FLOAT;
}
else if (fieldType.ToString().Equals("int") || fieldType.ToString().Equals("\"int\""))
{
columnType = Column.ColumnType.INT;
}
else if (fieldType.ToString().Equals("long") || fieldType.ToString().Equals("\"long\""))
{
columnType = Column.ColumnType.LONG;
}
else if (fieldType.ToString().Equals("string") || fieldType.ToString().Equals("\"string\""))
{
columnType = Column.ColumnType.STRING;
}
else
{
throw new ArgumentException("Field {fieldName} must be of type bytes, double, float, int, long or string.");
}
IList <string> columnProperties = null;
if (null != properties)
{
properties.TryGetValue(fieldName, out columnProperties);
}
// Check the column properties for nullability
if ((null != columnProperties) &&
(columnProperties.Contains(ColumnProperty.NULLABLE)))
{
is_column_nullable = true;
}
// Create the column to be added
Column column = new Column(fieldName, columnType, columnProperties);
// Set the "nullability"
column.setIsNullable(is_column_nullable);
m_data.columns.Add(column);
m_data.columnMap[fieldName] = m_data.columns.Count - 1;
}
} // end createSchemaFromString()
public opcSchemas()
{
stringType = (RecordSchema)RecordSchema.Parse(buildSchema("string"));
doubleType = (RecordSchema)RecordSchema.Parse(buildSchema("double"));
intType = (RecordSchema)RecordSchema.Parse(buildSchema("int"));
floatType = (RecordSchema)RecordSchema.Parse(buildSchema("float"));
booleanType = (RecordSchema)RecordSchema.Parse(buildSchema("boolean"));
longType = (RecordSchema)RecordSchema.Parse(buildSchema("long"));
rpcRequest = (RecordSchema)RecordSchema.Parse(@"{
'name': 'JSON_RPC_2_0_Request',
'type': 'record',
'fields': [
{
'name': 'method',
'type': 'string'
},
{
'name': 'params',
'type':['null',{'type':'array', 'items': 'string'}],
'default': null
},
{
'name': 'id',
'type': ['null','long'],
'default' : null
}
]
}");
rpcResponse = (RecordSchema)RecordSchema.Parse(@"
{
'name': 'JSON_RPC_2_0_Response',
'type': 'record',
'fields': [
{
'name': 'result',
'type': ['null','string'],
'default':null
},
{
'name': 'error',
'type':[
'null',
{
'type':'record',
'name':'error',
'fields': [
{'name':'code', 'type':'int'},
{'name':'message', 'type':'string'}
]
}
],
'default': null
},
{
'name': 'id',
'type': 'long'
}
]
}");
rpcError = (RecordSchema)RecordSchema.Parse(@"
{
'type':'record',
'name':'error',
'fields': [
{'name':'code', 'type':'int'},
{'name':'message', 'type':'string'}
]
}");
}